2024-11-16 14:14:02,742 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 14:14:02,758 main DEBUG Took 0.013805 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 14:14:02,759 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 14:14:02,759 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 14:14:02,761 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 14:14:02,762 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,773 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 14:14:02,789 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,791 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,792 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,793 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,794 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,794 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,796 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,796 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,797 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,799 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,799 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,800 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,801 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,802 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,803 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,804 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,806 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:14:02,807 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,807 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 14:14:02,809 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:14:02,811 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 14:14:02,814 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 14:14:02,815 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 14:14:02,817 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 14:14:02,818 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 14:14:02,828 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 14:14:02,832 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 14:14:02,834 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 14:14:02,835 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 14:14:02,835 main DEBUG createAppenders(={Console}) 2024-11-16 14:14:02,836 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-16 14:14:02,837 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 14:14:02,837 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-16 14:14:02,838 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 14:14:02,838 main DEBUG OutputStream closed 2024-11-16 14:14:02,839 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 14:14:02,839 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 14:14:02,839 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-16 14:14:02,914 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 14:14:02,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 14:14:02,917 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 14:14:02,919 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 14:14:02,919 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 14:14:02,920 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 14:14:02,920 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 14:14:02,920 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 14:14:02,921 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 14:14:02,921 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 14:14:02,921 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 14:14:02,921 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 14:14:02,922 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 14:14:02,922 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 14:14:02,922 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 14:14:02,923 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 14:14:02,923 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 14:14:02,924 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 14:14:02,926 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 14:14:02,926 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-16 14:14:02,927 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 14:14:02,927 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-16T14:14:03,133 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164 2024-11-16 14:14:03,136 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 14:14:03,137 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T14:14:03,147 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-16T14:14:03,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T14:14:03,188 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/cluster_786ab5e4-0d7c-0505-b9b1-bd85905ca37b, deleteOnExit=true 2024-11-16T14:14:03,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T14:14:03,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/test.cache.data in system properties and HBase conf 2024-11-16T14:14:03,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T14:14:03,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/hadoop.log.dir in system properties and HBase conf 2024-11-16T14:14:03,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T14:14:03,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T14:14:03,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T14:14:03,279 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T14:14:03,367 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T14:14:03,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T14:14:03,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T14:14:03,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T14:14:03,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T14:14:03,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T14:14:03,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T14:14:03,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T14:14:03,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T14:14:03,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T14:14:03,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/nfs.dump.dir in system properties and HBase conf 2024-11-16T14:14:03,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/java.io.tmpdir in system properties and HBase conf 2024-11-16T14:14:03,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T14:14:03,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T14:14:03,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T14:14:04,450 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T14:14:04,519 INFO [Time-limited test {}] log.Log(170): Logging initialized @2451ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T14:14:04,583 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:14:04,635 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T14:14:04,656 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T14:14:04,656 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T14:14:04,657 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T14:14:04,668 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:14:04,670 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/hadoop.log.dir/,AVAILABLE} 2024-11-16T14:14:04,671 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T14:14:04,841 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/java.io.tmpdir/jetty-localhost-37961-hadoop-hdfs-3_4_1-tests_jar-_-any-16103065913360421919/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T14:14:04,847 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:37961} 2024-11-16T14:14:04,847 INFO [Time-limited test {}] server.Server(415): Started @2779ms 2024-11-16T14:14:05,474 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:14:05,481 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T14:14:05,482 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T14:14:05,483 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T14:14:05,483 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T14:14:05,484 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/hadoop.log.dir/,AVAILABLE} 2024-11-16T14:14:05,484 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T14:14:05,585 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/java.io.tmpdir/jetty-localhost-41297-hadoop-hdfs-3_4_1-tests_jar-_-any-15757559914298198650/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:14:05,586 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:41297} 2024-11-16T14:14:05,586 INFO [Time-limited test {}] server.Server(415): Started @3518ms 2024-11-16T14:14:05,632 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T14:14:06,585 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/cluster_786ab5e4-0d7c-0505-b9b1-bd85905ca37b/data/data1/current/BP-517253520-172.17.0.2-1731766443902/current, will proceed with Du for space computation calculation, 2024-11-16T14:14:06,585 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/cluster_786ab5e4-0d7c-0505-b9b1-bd85905ca37b/data/data2/current/BP-517253520-172.17.0.2-1731766443902/current, will proceed with Du for space computation calculation, 2024-11-16T14:14:06,618 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T14:14:06,661 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1bc22dc64fc03ec with lease ID 0x858df32ba0f5041b: Processing first storage report for DS-59ff0f34-3a0c-4597-a8c8-bed378a70cfd from datanode DatanodeRegistration(127.0.0.1:38093, datanodeUuid=d94a9d10-33bc-48ed-8e36-884d4c3230f4, infoPort=36557, infoSecurePort=0, ipcPort=32857, storageInfo=lv=-57;cid=testClusterID;nsid=986539980;c=1731766443902) 2024-11-16T14:14:06,662 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1bc22dc64fc03ec with lease ID 0x858df32ba0f5041b: from storage DS-59ff0f34-3a0c-4597-a8c8-bed378a70cfd node DatanodeRegistration(127.0.0.1:38093, datanodeUuid=d94a9d10-33bc-48ed-8e36-884d4c3230f4, infoPort=36557, infoSecurePort=0, ipcPort=32857, storageInfo=lv=-57;cid=testClusterID;nsid=986539980;c=1731766443902), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T14:14:06,663 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1bc22dc64fc03ec with lease ID 0x858df32ba0f5041b: Processing first storage report for DS-81442f4d-2842-4bd8-9d0f-0aab8d5783f4 from datanode DatanodeRegistration(127.0.0.1:38093, datanodeUuid=d94a9d10-33bc-48ed-8e36-884d4c3230f4, infoPort=36557, infoSecurePort=0, ipcPort=32857, storageInfo=lv=-57;cid=testClusterID;nsid=986539980;c=1731766443902) 2024-11-16T14:14:06,663 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1bc22dc64fc03ec with lease ID 0x858df32ba0f5041b: from storage DS-81442f4d-2842-4bd8-9d0f-0aab8d5783f4 node DatanodeRegistration(127.0.0.1:38093, datanodeUuid=d94a9d10-33bc-48ed-8e36-884d4c3230f4, infoPort=36557, infoSecurePort=0, ipcPort=32857, storageInfo=lv=-57;cid=testClusterID;nsid=986539980;c=1731766443902), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T14:14:06,723 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164 2024-11-16T14:14:06,788 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/cluster_786ab5e4-0d7c-0505-b9b1-bd85905ca37b/zookeeper_0, clientPort=50174, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/cluster_786ab5e4-0d7c-0505-b9b1-bd85905ca37b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/cluster_786ab5e4-0d7c-0505-b9b1-bd85905ca37b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T14:14:06,797 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50174 2024-11-16T14:14:06,807 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:14:06,810 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:14:07,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741825_1001 (size=7) 2024-11-16T14:14:07,413 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc with version=8 2024-11-16T14:14:07,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/hbase-staging 2024-11-16T14:14:07,492 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T14:14:07,685 INFO [Time-limited test {}] client.ConnectionUtils(128): master/04e93f2b5fad:0 server-side Connection retries=45 2024-11-16T14:14:07,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:14:07,694 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T14:14:07,698 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T14:14:07,698 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:14:07,698 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T14:14:07,827 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T14:14:07,876 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T14:14:07,884 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T14:14:07,887 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:14:07,908 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4802 (auto-detected) 2024-11-16T14:14:07,909 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T14:14:07,926 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39231 2024-11-16T14:14:07,946 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39231 connecting to ZooKeeper ensemble=127.0.0.1:50174 2024-11-16T14:14:08,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:392310x0, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T14:14:08,093 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39231-0x10143fee65f0000 connected 2024-11-16T14:14:08,208 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:14:08,211 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:14:08,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:14:08,224 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc, hbase.cluster.distributed=false 2024-11-16T14:14:08,249 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T14:14:08,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39231 2024-11-16T14:14:08,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39231 2024-11-16T14:14:08,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39231 2024-11-16T14:14:08,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39231 2024-11-16T14:14:08,256 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39231 2024-11-16T14:14:08,373 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/04e93f2b5fad:0 server-side Connection retries=45 2024-11-16T14:14:08,375 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:14:08,375 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T14:14:08,375 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T14:14:08,375 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:14:08,375 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T14:14:08,378 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T14:14:08,380 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:14:08,381 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41317 2024-11-16T14:14:08,383 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41317 connecting to ZooKeeper ensemble=127.0.0.1:50174 2024-11-16T14:14:08,384 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:14:08,387 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:14:08,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413170x0, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T14:14:08,397 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:413170x0, quorum=127.0.0.1:50174, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:14:08,397 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41317-0x10143fee65f0001 connected 2024-11-16T14:14:08,400 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T14:14:08,407 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T14:14:08,410 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T14:14:08,414 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T14:14:08,415 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41317 2024-11-16T14:14:08,415 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41317 2024-11-16T14:14:08,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41317 2024-11-16T14:14:08,417 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41317 2024-11-16T14:14:08,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41317 2024-11-16T14:14:08,431 DEBUG [M:0;04e93f2b5fad:39231 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;04e93f2b5fad:39231 2024-11-16T14:14:08,432 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/04e93f2b5fad,39231,1731766447537 2024-11-16T14:14:08,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:14:08,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:14:08,452 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/04e93f2b5fad,39231,1731766447537 2024-11-16T14:14:08,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:14:08,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T14:14:08,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:14:08,481 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T14:14:08,482 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/04e93f2b5fad,39231,1731766447537 from backup master directory 2024-11-16T14:14:08,492 WARN [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T14:14:08,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/04e93f2b5fad,39231,1731766447537 2024-11-16T14:14:08,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:14:08,492 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=04e93f2b5fad,39231,1731766447537 2024-11-16T14:14:08,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:14:08,495 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T14:14:08,496 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T14:14:08,550 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/hbase.id] with ID: 22fd26c9-964e-434b-a1ee-372cf8e65a16 2024-11-16T14:14:08,550 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/.tmp/hbase.id 2024-11-16T14:14:08,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741826_1002 (size=42) 2024-11-16T14:14:08,961 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/.tmp/hbase.id]:[hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/hbase.id] 2024-11-16T14:14:09,012 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:14:09,017 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T14:14:09,039 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-16T14:14:09,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:14:09,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:14:09,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741827_1003 (size=196) 2024-11-16T14:14:09,494 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:14:09,497 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T14:14:09,510 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:09,514 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:14:09,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741828_1004 (size=1189) 2024-11-16T14:14:09,962 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store 2024-11-16T14:14:09,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741829_1005 (size=34) 2024-11-16T14:14:10,397 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T14:14:10,401 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:14:10,402 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T14:14:10,402 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:14:10,403 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:14:10,404 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T14:14:10,405 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:14:10,405 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:14:10,406 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731766450402Disabling compacts and flushes for region at 1731766450402Disabling writes for close at 1731766450404 (+2 ms)Writing region close event to WAL at 1731766450405 (+1 ms)Closed at 1731766450405 2024-11-16T14:14:10,408 WARN [master/04e93f2b5fad:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/.initializing 2024-11-16T14:14:10,409 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/WALs/04e93f2b5fad,39231,1731766447537 2024-11-16T14:14:10,417 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:14:10,431 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=04e93f2b5fad%2C39231%2C1731766447537, suffix=, logDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/WALs/04e93f2b5fad,39231,1731766447537, archiveDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/oldWALs, maxLogs=10 2024-11-16T14:14:10,451 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/WALs/04e93f2b5fad,39231,1731766447537/04e93f2b5fad%2C39231%2C1731766447537.1731766450435, exclude list is [], retry=0 2024-11-16T14:14:10,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38093,DS-59ff0f34-3a0c-4597-a8c8-bed378a70cfd,DISK] 2024-11-16T14:14:10,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T14:14:10,503 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/WALs/04e93f2b5fad,39231,1731766447537/04e93f2b5fad%2C39231%2C1731766447537.1731766450435 2024-11-16T14:14:10,504 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-16T14:14:10,505 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:14:10,505 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:14:10,509 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:14:10,510 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:14:10,542 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:14:10,563 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T14:14:10,566 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:10,569 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:14:10,569 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:14:10,573 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T14:14:10,573 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:10,574 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:14:10,574 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:14:10,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T14:14:10,577 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:10,578 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:14:10,578 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:14:10,580 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T14:14:10,581 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:10,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:14:10,582 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:14:10,585 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:14:10,587 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:14:10,591 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:14:10,592 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:14:10,595 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T14:14:10,599 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:14:10,604 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:14:10,605 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72914834, jitterRate=0.08651569485664368}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T14:14:10,611 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731766450520Initializing all the Stores at 1731766450522 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731766450523 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766450523Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766450524 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766450524Cleaning up temporary data from old regions at 1731766450592 (+68 ms)Region opened successfully at 1731766450611 (+19 ms) 2024-11-16T14:14:10,612 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T14:14:10,644 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794617bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=04e93f2b5fad/172.17.0.2:0 2024-11-16T14:14:10,672 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T14:14:10,681 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T14:14:10,681 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T14:14:10,683 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T14:14:10,684 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T14:14:10,689 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T14:14:10,689 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T14:14:10,711 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T14:14:10,719 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T14:14:10,810 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T14:14:10,815 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T14:14:10,818 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T14:14:10,896 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T14:14:10,903 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T14:14:10,910 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T14:14:10,991 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T14:14:10,997 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T14:14:11,054 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T14:14:11,076 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T14:14:11,143 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T14:14:11,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T14:14:11,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T14:14:11,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:14:11,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:14:11,302 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=04e93f2b5fad,39231,1731766447537, sessionid=0x10143fee65f0000, setting cluster-up flag (Was=false) 2024-11-16T14:14:11,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:14:11,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:14:11,775 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T14:14:11,779 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=04e93f2b5fad,39231,1731766447537 2024-11-16T14:14:11,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:14:11,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:14:12,028 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T14:14:12,033 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=04e93f2b5fad,39231,1731766447537 2024-11-16T14:14:12,044 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T14:14:12,112 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T14:14:12,122 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T14:14:12,125 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(746): ClusterId : 22fd26c9-964e-434b-a1ee-372cf8e65a16 2024-11-16T14:14:12,128 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T14:14:12,131 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T14:14:12,137 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 04e93f2b5fad,39231,1731766447537 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T14:14:12,145 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T14:14:12,145 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T14:14:12,145 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/04e93f2b5fad:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:14:12,146 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/04e93f2b5fad:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:14:12,146 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/04e93f2b5fad:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:14:12,146 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/04e93f2b5fad:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:14:12,146 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/04e93f2b5fad:0, corePoolSize=10, maxPoolSize=10 2024-11-16T14:14:12,146 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,147 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/04e93f2b5fad:0, corePoolSize=2, maxPoolSize=2 2024-11-16T14:14:12,147 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,150 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731766482150 2024-11-16T14:14:12,152 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T14:14:12,153 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T14:14:12,153 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T14:14:12,154 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T14:14:12,155 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T14:14:12,155 DEBUG [RS:0;04e93f2b5fad:41317 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a87c644, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=04e93f2b5fad/172.17.0.2:0 2024-11-16T14:14:12,157 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T14:14:12,158 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T14:14:12,158 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T14:14:12,158 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T14:14:12,159 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:12,159 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T14:14:12,160 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,164 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T14:14:12,166 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T14:14:12,166 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T14:14:12,169 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T14:14:12,170 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T14:14:12,175 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/04e93f2b5fad:0:becomeActiveMaster-HFileCleaner.large.0-1731766452171,5,FailOnTimeoutGroup] 2024-11-16T14:14:12,175 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;04e93f2b5fad:41317 2024-11-16T14:14:12,175 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/04e93f2b5fad:0:becomeActiveMaster-HFileCleaner.small.0-1731766452175,5,FailOnTimeoutGroup] 2024-11-16T14:14:12,176 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,176 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T14:14:12,177 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741831_1007 (size=1321) 2024-11-16T14:14:12,178 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,179 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T14:14:12,179 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T14:14:12,179 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T14:14:12,182 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(2659): reportForDuty to master=04e93f2b5fad,39231,1731766447537 with port=41317, startcode=1731766448342 2024-11-16T14:14:12,193 DEBUG [RS:0;04e93f2b5fad:41317 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T14:14:12,225 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52019, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T14:14:12,233 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39231 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:12,235 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39231 {}] master.ServerManager(517): Registering regionserver=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:12,249 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc 2024-11-16T14:14:12,249 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43297 2024-11-16T14:14:12,250 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T14:14:12,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T14:14:12,342 DEBUG [RS:0;04e93f2b5fad:41317 {}] zookeeper.ZKUtil(111): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:12,342 WARN [RS:0;04e93f2b5fad:41317 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T14:14:12,342 INFO [RS:0;04e93f2b5fad:41317 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:14:12,343 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/WALs/04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:12,345 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [04e93f2b5fad,41317,1731766448342] 2024-11-16T14:14:12,366 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T14:14:12,384 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T14:14:12,388 INFO [RS:0;04e93f2b5fad:41317 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T14:14:12,388 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,389 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T14:14:12,396 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T14:14:12,398 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,398 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,398 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,399 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,399 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,399 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,399 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/04e93f2b5fad:0, corePoolSize=2, maxPoolSize=2 2024-11-16T14:14:12,399 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,400 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,400 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,400 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,401 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,401 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/04e93f2b5fad:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:14:12,401 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/04e93f2b5fad:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:14:12,401 DEBUG [RS:0;04e93f2b5fad:41317 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:14:12,402 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,403 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,403 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,403 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,403 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,403 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,41317,1731766448342-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T14:14:12,428 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T14:14:12,430 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,41317,1731766448342-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,430 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,430 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.Replication(171): 04e93f2b5fad,41317,1731766448342 started 2024-11-16T14:14:12,457 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:12,457 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1482): Serving as 04e93f2b5fad,41317,1731766448342, RpcServer on 04e93f2b5fad/172.17.0.2:41317, sessionid=0x10143fee65f0001 2024-11-16T14:14:12,459 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T14:14:12,459 DEBUG [RS:0;04e93f2b5fad:41317 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:12,459 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '04e93f2b5fad,41317,1731766448342' 2024-11-16T14:14:12,460 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T14:14:12,461 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T14:14:12,462 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T14:14:12,462 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T14:14:12,462 DEBUG [RS:0;04e93f2b5fad:41317 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:12,462 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '04e93f2b5fad,41317,1731766448342' 2024-11-16T14:14:12,462 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T14:14:12,463 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T14:14:12,464 DEBUG [RS:0;04e93f2b5fad:41317 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T14:14:12,464 INFO [RS:0;04e93f2b5fad:41317 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T14:14:12,464 INFO [RS:0;04e93f2b5fad:41317 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T14:14:12,570 INFO [RS:0;04e93f2b5fad:41317 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:14:12,574 INFO [RS:0;04e93f2b5fad:41317 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=04e93f2b5fad%2C41317%2C1731766448342, suffix=, logDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/WALs/04e93f2b5fad,41317,1731766448342, archiveDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/oldWALs, maxLogs=32 2024-11-16T14:14:12,580 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T14:14:12,581 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc 2024-11-16T14:14:12,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741832_1008 (size=32) 2024-11-16T14:14:12,595 DEBUG [RS:0;04e93f2b5fad:41317 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/WALs/04e93f2b5fad,41317,1731766448342/04e93f2b5fad%2C41317%2C1731766448342.1731766452578, exclude list is [], retry=0 2024-11-16T14:14:12,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38093,DS-59ff0f34-3a0c-4597-a8c8-bed378a70cfd,DISK] 2024-11-16T14:14:12,603 INFO [RS:0;04e93f2b5fad:41317 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/WALs/04e93f2b5fad,41317,1731766448342/04e93f2b5fad%2C41317%2C1731766448342.1731766452578 2024-11-16T14:14:12,603 DEBUG [RS:0;04e93f2b5fad:41317 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-16T14:14:12,993 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:14:12,998 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T14:14:13,001 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T14:14:13,001 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:13,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:14:13,003 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T14:14:13,006 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T14:14:13,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:13,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:14:13,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T14:14:13,011 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T14:14:13,011 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:13,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:14:13,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T14:14:13,015 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T14:14:13,015 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:13,016 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:14:13,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T14:14:13,018 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740 2024-11-16T14:14:13,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740 2024-11-16T14:14:13,023 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T14:14:13,023 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T14:14:13,024 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:14:13,027 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T14:14:13,031 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:14:13,032 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75414200, jitterRate=0.12375915050506592}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:14:13,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731766452994Initializing all the Stores at 1731766452997 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731766452997Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731766452998 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766452998Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731766452998Cleaning up temporary data from old regions at 1731766453023 (+25 ms)Region opened successfully at 1731766453035 (+12 ms) 2024-11-16T14:14:13,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T14:14:13,035 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T14:14:13,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T14:14:13,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T14:14:13,036 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T14:14:13,037 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T14:14:13,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731766453035Disabling compacts and flushes for region at 1731766453035Disabling writes for close at 1731766453036 (+1 ms)Writing region close event to WAL at 1731766453037 (+1 ms)Closed at 1731766453037 2024-11-16T14:14:13,045 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T14:14:13,045 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T14:14:13,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T14:14:13,062 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T14:14:13,066 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T14:14:13,219 DEBUG [04e93f2b5fad:39231 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-16T14:14:13,232 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:13,239 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 04e93f2b5fad,41317,1731766448342, state=OPENING 2024-11-16T14:14:13,258 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T14:14:13,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:14:13,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:14:13,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:14:13,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:14:13,272 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T14:14:13,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:14:13,457 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T14:14:13,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55683, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T14:14:13,468 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T14:14:13,469 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:14:13,469 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T14:14:13,472 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=04e93f2b5fad%2C41317%2C1731766448342.meta, suffix=.meta, logDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/WALs/04e93f2b5fad,41317,1731766448342, archiveDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/oldWALs, maxLogs=32 2024-11-16T14:14:13,487 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/WALs/04e93f2b5fad,41317,1731766448342/04e93f2b5fad%2C41317%2C1731766448342.meta.1731766453474.meta, exclude list is [], retry=0 2024-11-16T14:14:13,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38093,DS-59ff0f34-3a0c-4597-a8c8-bed378a70cfd,DISK] 2024-11-16T14:14:13,493 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/WALs/04e93f2b5fad,41317,1731766448342/04e93f2b5fad%2C41317%2C1731766448342.meta.1731766453474.meta 2024-11-16T14:14:13,494 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36557:36557)] 2024-11-16T14:14:13,495 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:14:13,497 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T14:14:13,501 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T14:14:13,505 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T14:14:13,510 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T14:14:13,510 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:14:13,511 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T14:14:13,511 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T14:14:13,514 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T14:14:13,515 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T14:14:13,516 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:13,516 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:14:13,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T14:14:13,518 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T14:14:13,518 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:13,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:14:13,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T14:14:13,521 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T14:14:13,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:13,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:14:13,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T14:14:13,524 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T14:14:13,524 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:13,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:14:13,525 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T14:14:13,527 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740 2024-11-16T14:14:13,530 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740 2024-11-16T14:14:13,532 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T14:14:13,532 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T14:14:13,533 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:14:13,536 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T14:14:13,537 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64534701, jitterRate=-0.038358017802238464}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:14:13,538 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T14:14:13,539 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731766453511Writing region info on filesystem at 1731766453511Initializing all the Stores at 1731766453513 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731766453513Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731766453514 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766453514Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731766453514Cleaning up temporary data from old regions at 1731766453532 (+18 ms)Running coprocessor post-open hooks at 1731766453538 (+6 ms)Region opened successfully at 1731766453539 (+1 ms) 2024-11-16T14:14:13,545 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731766453450 2024-11-16T14:14:13,555 DEBUG [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T14:14:13,556 INFO [RS_OPEN_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T14:14:13,558 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:13,560 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 04e93f2b5fad,41317,1731766448342, state=OPEN 2024-11-16T14:14:13,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:14:13,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:14:13,601 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:14:13,601 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:14:13,601 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:13,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T14:14:13,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=04e93f2b5fad,41317,1731766448342 in 326 msec 2024-11-16T14:14:13,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T14:14:13,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 560 msec 2024-11-16T14:14:13,619 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T14:14:13,619 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T14:14:13,638 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:13,640 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:13,658 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:13,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43133, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:13,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6110 sec 2024-11-16T14:14:13,685 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731766453685, completionTime=-1 2024-11-16T14:14:13,689 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-16T14:14:13,689 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-16T14:14:13,718 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-16T14:14:13,718 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731766513718 2024-11-16T14:14:13,718 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731766573718 2024-11-16T14:14:13,719 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 29 msec 2024-11-16T14:14:13,721 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,39231,1731766447537-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:13,722 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,39231,1731766447537-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:13,722 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,39231,1731766447537-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:13,724 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-04e93f2b5fad:39231, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:13,724 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:13,725 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:13,730 DEBUG [master/04e93f2b5fad:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T14:14:13,755 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 5.262sec 2024-11-16T14:14:13,756 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T14:14:13,758 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T14:14:13,759 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T14:14:13,759 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T14:14:13,759 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T14:14:13,760 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,39231,1731766447537-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T14:14:13,761 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,39231,1731766447537-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T14:14:13,770 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T14:14:13,771 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T14:14:13,772 INFO [master/04e93f2b5fad:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=04e93f2b5fad,39231,1731766447537-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:14:13,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:13,837 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T14:14:13,837 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T14:14:13,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:13,842 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:13,849 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:13,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775efbc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:13,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:13,859 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:13,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:13,862 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:13,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:13,891 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=04e93f2b5fad,39231,1731766447537 2024-11-16T14:14:13,901 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=237, ProcessCount=11, AvailableMemoryMB=4822 2024-11-16T14:14:13,908 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:14:13,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 04e93f2b5fad,39231,1731766447537 2024-11-16T14:14:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c35b008 2024-11-16T14:14:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:14:13,932 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:14:13,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:14:13,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:14:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T14:14:13,952 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:14:13,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-16T14:14:13,954 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:13,957 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:14:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:14:13,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741835_1011 (size=1061) 2024-11-16T14:14:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:14:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:14:14,382 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc 2024-11-16T14:14:14,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741836_1012 (size=53) 2024-11-16T14:14:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:14:14,796 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:14:14,796 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fd241767f9d916c6c1f6b781eb2b8ff7, disabling compactions & flushes 2024-11-16T14:14:14,797 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:14,797 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:14,797 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. after waiting 0 ms 2024-11-16T14:14:14,797 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:14,798 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:14,798 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fd241767f9d916c6c1f6b781eb2b8ff7: Waiting for close lock at 1731766454796Disabling compacts and flushes for region at 1731766454796Disabling writes for close at 1731766454797 (+1 ms)Writing region close event to WAL at 1731766454797Closed at 1731766454797 2024-11-16T14:14:14,803 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:14:14,809 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731766454803"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731766454803"}]},"ts":"1731766454803"} 2024-11-16T14:14:14,814 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:14:14,816 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:14:14,819 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766454816"}]},"ts":"1731766454816"} 2024-11-16T14:14:14,823 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T14:14:14,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fd241767f9d916c6c1f6b781eb2b8ff7, ASSIGN}] 2024-11-16T14:14:14,827 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fd241767f9d916c6c1f6b781eb2b8ff7, ASSIGN 2024-11-16T14:14:14,829 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fd241767f9d916c6c1f6b781eb2b8ff7, ASSIGN; state=OFFLINE, location=04e93f2b5fad,41317,1731766448342; forceNewPlan=false, retain=false 2024-11-16T14:14:14,981 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fd241767f9d916c6c1f6b781eb2b8ff7, regionState=OPENING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:14,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fd241767f9d916c6c1f6b781eb2b8ff7, ASSIGN because future has completed 2024-11-16T14:14:14,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:14:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:14:15,153 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:15,153 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:14:15,154 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,154 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:14:15,154 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,154 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,156 INFO [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,159 INFO [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:14:15,160 INFO [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd241767f9d916c6c1f6b781eb2b8ff7 columnFamilyName A 2024-11-16T14:14:15,160 DEBUG [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:15,161 INFO [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] regionserver.HStore(327): Store=fd241767f9d916c6c1f6b781eb2b8ff7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:14:15,161 INFO [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,163 INFO [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:14:15,164 INFO [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd241767f9d916c6c1f6b781eb2b8ff7 columnFamilyName B 2024-11-16T14:14:15,164 DEBUG [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:15,165 INFO [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] regionserver.HStore(327): Store=fd241767f9d916c6c1f6b781eb2b8ff7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:14:15,165 INFO [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,167 INFO [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:14:15,168 INFO [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd241767f9d916c6c1f6b781eb2b8ff7 columnFamilyName C 2024-11-16T14:14:15,168 DEBUG [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:15,169 INFO [StoreOpener-fd241767f9d916c6c1f6b781eb2b8ff7-1 {}] regionserver.HStore(327): Store=fd241767f9d916c6c1f6b781eb2b8ff7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:14:15,169 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:15,169 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,171 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,171 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,173 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,173 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,174 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:14:15,176 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,180 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:14:15,181 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fd241767f9d916c6c1f6b781eb2b8ff7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70361810, jitterRate=0.04847267270088196}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:14:15,181 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:15,182 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fd241767f9d916c6c1f6b781eb2b8ff7: Running coprocessor pre-open hook at 1731766455154Writing region info on filesystem at 1731766455154Initializing all the Stores at 1731766455156 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766455156Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766455156Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766455156Cleaning up temporary data from old regions at 1731766455173 (+17 ms)Running coprocessor post-open hooks at 1731766455181 (+8 ms)Region opened successfully at 1731766455182 (+1 ms) 2024-11-16T14:14:15,184 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., pid=6, masterSystemTime=1731766455147 2024-11-16T14:14:15,187 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:15,187 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:15,189 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fd241767f9d916c6c1f6b781eb2b8ff7, regionState=OPEN, openSeqNum=2, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:15,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:14:15,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T14:14:15,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 in 204 msec 2024-11-16T14:14:15,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T14:14:15,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fd241767f9d916c6c1f6b781eb2b8ff7, ASSIGN in 376 msec 2024-11-16T14:14:15,207 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:14:15,207 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766455207"}]},"ts":"1731766455207"} 2024-11-16T14:14:15,211 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T14:14:15,212 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:14:15,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2700 sec 2024-11-16T14:14:16,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:14:16,129 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:16,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,137 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:16,137 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:16,138 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:16,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:16,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:16,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bff7409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:16,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:16,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:16,141 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2538dc8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,143 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:16,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:16,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:16,149 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:14:16,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 04e93f2b5fad,39231,1731766447537 2024-11-16T14:14:16,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@481dbf66 2024-11-16T14:14:16,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:14:16,156 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:14:16,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:16,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:16,162 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:16,165 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:16,167 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:16,169 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:16,171 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:16,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:16,175 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:16,179 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:16,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:16,183 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5367b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:16,186 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b3fb7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:16,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:16,190 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4331d0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:16,192 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:16,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17daddbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:16,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:16,197 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:16,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60486904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:16,200 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a2dcb68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,202 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:16,202 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:16,203 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b8551d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:16,205 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:16,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517e68dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:16,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:16,209 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c1b5617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:16,211 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:16,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425be669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:16,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:16,214 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@552f3363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:16,217 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:16,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33abc70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:16,222 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:16,222 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:16,222 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:16,222 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:16,223 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:16,224 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:16,224 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:16,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:16,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:16,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:16,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:16,225 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:16,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:16,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:16,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:16,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:16,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:16,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:16,233 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:14:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:14:16,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:14:16,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:16,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:14:16,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:16,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:14:16,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:14:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:14:16,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:14:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:14:16,266 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-16T14:14:16,276 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:14:16,278 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:16,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:16,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:14:16,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:16,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:16,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:16,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:16,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:16,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:14:16,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766516414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766516423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766516425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766516425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766516430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:14:16,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:16,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:16,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/1035b488445e42c49800699e880594e2 is 50, key is test_row_0/A:col10/1731766456307/Put/seqid=0 2024-11-16T14:14:16,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:16,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:16,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741837_1013 (size=14341) 2024-11-16T14:14:16,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766516578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766516579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766516579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766516578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766516579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:14:16,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:14:16,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:16,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:16,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:16,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:16,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:16,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:14:16,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766516787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766516787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:16,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766516796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766516797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766516798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:16,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:16,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/1035b488445e42c49800699e880594e2 2024-11-16T14:14:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:14:16,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:14:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:16,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:16,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/3383694bc41244079f5efd3e292ae90e is 50, key is test_row_0/B:col10/1731766456307/Put/seqid=0 2024-11-16T14:14:17,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741838_1014 (size=12001) 2024-11-16T14:14:17,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/3383694bc41244079f5efd3e292ae90e 2024-11-16T14:14:17,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/504adb91d46b48fc8c6c690adc8ebe61 is 50, key is test_row_0/C:col10/1731766456307/Put/seqid=0 2024-11-16T14:14:17,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741839_1015 (size=12001) 2024-11-16T14:14:17,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/504adb91d46b48fc8c6c690adc8ebe61 2024-11-16T14:14:17,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/1035b488445e42c49800699e880594e2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/1035b488445e42c49800699e880594e2 2024-11-16T14:14:17,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/1035b488445e42c49800699e880594e2, entries=200, sequenceid=14, filesize=14.0 K 2024-11-16T14:14:17,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/3383694bc41244079f5efd3e292ae90e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/3383694bc41244079f5efd3e292ae90e 2024-11-16T14:14:17,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/3383694bc41244079f5efd3e292ae90e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T14:14:17,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:14:17,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:17,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:17,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:17,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/504adb91d46b48fc8c6c690adc8ebe61 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/504adb91d46b48fc8c6c690adc8ebe61 2024-11-16T14:14:17,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/504adb91d46b48fc8c6c690adc8ebe61, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T14:14:17,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for fd241767f9d916c6c1f6b781eb2b8ff7 in 820ms, sequenceid=14, compaction requested=false 2024-11-16T14:14:17,157 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T14:14:17,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:17,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731766456415, max timestamp=1731766457153Num uniques 859; 2024-11-16T14:14:17,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:14:17,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T14:14:17,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:17,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766456415, max timestamp=1731766457153Num uniques 859; 2024-11-16T14:14:17,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:17,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:14:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:17,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/18f0c92bc5c04aeebcc2b16f3ed7bbac is 50, key is test_row_0/A:col10/1731766457147/Put/seqid=0 2024-11-16T14:14:17,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741840_1016 (size=12001) 2024-11-16T14:14:17,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/18f0c92bc5c04aeebcc2b16f3ed7bbac 2024-11-16T14:14:17,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/335df2fa2c1d4c868640163c9a905aa2 is 50, key is test_row_0/B:col10/1731766457147/Put/seqid=0 2024-11-16T14:14:17,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:14:17,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:17,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:17,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741841_1017 (size=12001) 2024-11-16T14:14:17,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/335df2fa2c1d4c868640163c9a905aa2 2024-11-16T14:14:17,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/49ba03208fb34c0486f00357272efc55 is 50, key is test_row_0/C:col10/1731766457147/Put/seqid=0 2024-11-16T14:14:17,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741842_1018 (size=12001) 2024-11-16T14:14:17,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/49ba03208fb34c0486f00357272efc55 2024-11-16T14:14:17,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/18f0c92bc5c04aeebcc2b16f3ed7bbac as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/18f0c92bc5c04aeebcc2b16f3ed7bbac 2024-11-16T14:14:17,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766517297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766517308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/18f0c92bc5c04aeebcc2b16f3ed7bbac, entries=150, sequenceid=36, filesize=11.7 K 2024-11-16T14:14:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766517347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766517357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/335df2fa2c1d4c868640163c9a905aa2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/335df2fa2c1d4c868640163c9a905aa2 2024-11-16T14:14:17,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766517367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/335df2fa2c1d4c868640163c9a905aa2, entries=150, sequenceid=36, filesize=11.7 K 2024-11-16T14:14:17,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/49ba03208fb34c0486f00357272efc55 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/49ba03208fb34c0486f00357272efc55 2024-11-16T14:14:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:14:17,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/49ba03208fb34c0486f00357272efc55, entries=150, sequenceid=36, filesize=11.7 K 2024-11-16T14:14:17,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:14:17,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:17,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:17,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:17,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~69.33 KB/70990, heapSize ~194.20 KB/198864, currentSize=107.34 KB/109920 for fd241767f9d916c6c1f6b781eb2b8ff7 in 255ms, sequenceid=36, compaction requested=false 2024-11-16T14:14:17,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:17,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:17,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-16T14:14:17,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:17,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:17,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:17,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:17,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:17,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:17,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/6e7a49004a6c42a7a7dac9cba330122d is 50, key is test_row_0/A:col10/1731766457506/Put/seqid=0 2024-11-16T14:14:17,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741843_1019 (size=12001) 2024-11-16T14:14:17,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/6e7a49004a6c42a7a7dac9cba330122d 2024-11-16T14:14:17,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:14:17,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:17,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:17,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:17,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/1878281c4422420182487c8a15de1290 is 50, key is test_row_0/B:col10/1731766457506/Put/seqid=0 2024-11-16T14:14:17,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766517551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766517563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766517573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766517591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766517584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741844_1020 (size=12001) 2024-11-16T14:14:17,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766517706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766517721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766517722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766517722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766517722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:14:17,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:17,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:17,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:17,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,873 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T14:14:17,874 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T14:14:17,882 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T14:14:17,882 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T14:14:17,883 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T14:14:17,883 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T14:14:17,884 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T14:14:17,884 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T14:14:17,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:14:17,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:17,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:17,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:17,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:17,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766517926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766517936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766517937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766517937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766517937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/1878281c4422420182487c8a15de1290 2024-11-16T14:14:18,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:14:18,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:18,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:18,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:18,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:18,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:18,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/ff42ac1eb41c491288852ada8abffbf4 is 50, key is test_row_0/C:col10/1731766457506/Put/seqid=0 2024-11-16T14:14:18,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741845_1021 (size=12001) 2024-11-16T14:14:18,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/ff42ac1eb41c491288852ada8abffbf4 2024-11-16T14:14:18,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/6e7a49004a6c42a7a7dac9cba330122d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/6e7a49004a6c42a7a7dac9cba330122d 2024-11-16T14:14:18,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/6e7a49004a6c42a7a7dac9cba330122d, entries=150, sequenceid=58, filesize=11.7 K 2024-11-16T14:14:18,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/1878281c4422420182487c8a15de1290 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/1878281c4422420182487c8a15de1290 2024-11-16T14:14:18,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/1878281c4422420182487c8a15de1290, entries=150, sequenceid=58, filesize=11.7 K 2024-11-16T14:14:18,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/ff42ac1eb41c491288852ada8abffbf4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/ff42ac1eb41c491288852ada8abffbf4 2024-11-16T14:14:18,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/ff42ac1eb41c491288852ada8abffbf4, entries=150, sequenceid=58, filesize=11.7 K 2024-11-16T14:14:18,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for fd241767f9d916c6c1f6b781eb2b8ff7 in 647ms, sequenceid=58, compaction requested=true 2024-11-16T14:14:18,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:18,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:18,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:18,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:18,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:18,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:18,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:18,164 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:18,164 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:18,168 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:18,170 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/B is initiating minor compaction (all files) 2024-11-16T14:14:18,171 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/B in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:18,171 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/3383694bc41244079f5efd3e292ae90e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/335df2fa2c1d4c868640163c9a905aa2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/1878281c4422420182487c8a15de1290] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=35.2 K 2024-11-16T14:14:18,173 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3383694bc41244079f5efd3e292ae90e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731766456295 2024-11-16T14:14:18,174 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 335df2fa2c1d4c868640163c9a905aa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731766456420 2024-11-16T14:14:18,175 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1878281c4422420182487c8a15de1290, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731766457356 2024-11-16T14:14:18,184 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:18,184 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/A is initiating minor compaction (all files) 2024-11-16T14:14:18,184 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/A in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:18,185 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/1035b488445e42c49800699e880594e2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/18f0c92bc5c04aeebcc2b16f3ed7bbac, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/6e7a49004a6c42a7a7dac9cba330122d] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=37.4 K 2024-11-16T14:14:18,192 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1035b488445e42c49800699e880594e2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731766456283 2024-11-16T14:14:18,193 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18f0c92bc5c04aeebcc2b16f3ed7bbac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731766456420 2024-11-16T14:14:18,195 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e7a49004a6c42a7a7dac9cba330122d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731766457356 2024-11-16T14:14:18,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:14:18,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:18,210 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:14:18,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:18,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:18,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:18,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/ed579eab3d9149dbaaddbf612317985d is 50, key is test_row_0/A:col10/1731766457572/Put/seqid=0 2024-11-16T14:14:18,241 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#B#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:18,242 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/7f1394528b544e90abf0d65c242aba45 is 50, key is test_row_0/B:col10/1731766457506/Put/seqid=0 2024-11-16T14:14:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:18,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:18,250 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#A#compaction#11 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:18,251 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/4dd357149d7c47998ac7f464b9a91899 is 50, key is test_row_0/A:col10/1731766457506/Put/seqid=0 2024-11-16T14:14:18,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741846_1022 (size=9657) 2024-11-16T14:14:18,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741848_1024 (size=12104) 2024-11-16T14:14:18,290 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/ed579eab3d9149dbaaddbf612317985d 2024-11-16T14:14:18,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741847_1023 (size=12104) 2024-11-16T14:14:18,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/a0b7109a7fd348b287ee976eb5c57922 is 50, key is test_row_0/B:col10/1731766457572/Put/seqid=0 2024-11-16T14:14:18,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741849_1025 (size=9657) 2024-11-16T14:14:18,330 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/a0b7109a7fd348b287ee976eb5c57922 2024-11-16T14:14:18,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731766458268, max timestamp=1731766458328Num uniques 859; 2024-11-16T14:14:18,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-16T14:14:18,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731766458268, max timestamp=1731766458328Num uniques 859; 2024-11-16T14:14:18,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-16T14:14:18,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/1d760dab85f64acea5d74919927b72e0 is 50, key is test_row_0/C:col10/1731766457572/Put/seqid=0 2024-11-16T14:14:18,360 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T14:14:18,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741850_1026 (size=9657) 2024-11-16T14:14:18,380 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/1d760dab85f64acea5d74919927b72e0 2024-11-16T14:14:18,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766518368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766518365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/ed579eab3d9149dbaaddbf612317985d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed579eab3d9149dbaaddbf612317985d 2024-11-16T14:14:18,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766518390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766518392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766518392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,418 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed579eab3d9149dbaaddbf612317985d, entries=100, sequenceid=68, filesize=9.4 K 2024-11-16T14:14:18,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/a0b7109a7fd348b287ee976eb5c57922 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/a0b7109a7fd348b287ee976eb5c57922 2024-11-16T14:14:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:14:18,437 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/a0b7109a7fd348b287ee976eb5c57922, entries=100, sequenceid=68, filesize=9.4 K 2024-11-16T14:14:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/1d760dab85f64acea5d74919927b72e0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1d760dab85f64acea5d74919927b72e0 2024-11-16T14:14:18,461 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1d760dab85f64acea5d74919927b72e0, entries=100, sequenceid=68, filesize=9.4 K 2024-11-16T14:14:18,464 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=145.36 KB/148850 for fd241767f9d916c6c1f6b781eb2b8ff7 in 254ms, sequenceid=68, compaction requested=true 2024-11-16T14:14:18,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:18,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:18,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-16T14:14:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-16T14:14:18,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T14:14:18,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1880 sec 2024-11-16T14:14:18,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.2070 sec 2024-11-16T14:14:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:18,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=152.07 KB heapSize=424.33 KB 2024-11-16T14:14:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:18,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:18,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/366f065c0c514b9d8ccea2b384e6f799 is 50, key is test_row_0/A:col10/1731766458353/Put/seqid=0 2024-11-16T14:14:18,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766518518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766518518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766518523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766518525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766518529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741851_1027 (size=12001) 2024-11-16T14:14:18,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/366f065c0c514b9d8ccea2b384e6f799 2024-11-16T14:14:18,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/8795b11a4c64465cb888fbbb10fa293f is 50, key is test_row_0/B:col10/1731766458353/Put/seqid=0 2024-11-16T14:14:18,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741852_1028 (size=12001) 2024-11-16T14:14:18,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/8795b11a4c64465cb888fbbb10fa293f 2024-11-16T14:14:18,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/a8d7510ef01f41298504184a00922dee is 50, key is test_row_0/C:col10/1731766458353/Put/seqid=0 2024-11-16T14:14:18,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766518637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766518637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766518637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741853_1029 (size=12001) 2024-11-16T14:14:18,709 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/7f1394528b544e90abf0d65c242aba45 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7f1394528b544e90abf0d65c242aba45 2024-11-16T14:14:18,711 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/4dd357149d7c47998ac7f464b9a91899 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/4dd357149d7c47998ac7f464b9a91899 2024-11-16T14:14:18,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766518736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766518737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,752 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/B of fd241767f9d916c6c1f6b781eb2b8ff7 into 7f1394528b544e90abf0d65c242aba45(size=11.8 K), total size for store is 21.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:18,752 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:18,753 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/A of fd241767f9d916c6c1f6b781eb2b8ff7 into 4dd357149d7c47998ac7f464b9a91899(size=11.8 K), total size for store is 21.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:18,753 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:18,756 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/A, priority=13, startTime=1731766458160; duration=0sec 2024-11-16T14:14:18,757 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:18,757 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:18,758 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:14:18,758 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/B, priority=13, startTime=1731766458163; duration=0sec 2024-11-16T14:14:18,759 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:18,759 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:18,762 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:14:18,762 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/C is initiating minor compaction (all files) 2024-11-16T14:14:18,763 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/C in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:18,763 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/504adb91d46b48fc8c6c690adc8ebe61, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/49ba03208fb34c0486f00357272efc55, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/ff42ac1eb41c491288852ada8abffbf4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1d760dab85f64acea5d74919927b72e0] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=44.6 K 2024-11-16T14:14:18,764 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 504adb91d46b48fc8c6c690adc8ebe61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731766456295 2024-11-16T14:14:18,765 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49ba03208fb34c0486f00357272efc55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731766456420 2024-11-16T14:14:18,766 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff42ac1eb41c491288852ada8abffbf4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731766457356 2024-11-16T14:14:18,767 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d760dab85f64acea5d74919927b72e0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731766457572 2024-11-16T14:14:18,814 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#C#compaction#17 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:18,815 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/0ac3cab406e54a508d25feecfa1660e5 is 50, key is test_row_0/C:col10/1731766457572/Put/seqid=0 2024-11-16T14:14:18,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741854_1030 (size=12139) 2024-11-16T14:14:18,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766518856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766518856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766518856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766519056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766519056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/a8d7510ef01f41298504184a00922dee 2024-11-16T14:14:19,072 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:14:19,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/366f065c0c514b9d8ccea2b384e6f799 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/366f065c0c514b9d8ccea2b384e6f799 2024-11-16T14:14:19,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/366f065c0c514b9d8ccea2b384e6f799, entries=150, sequenceid=99, filesize=11.7 K 2024-11-16T14:14:19,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/8795b11a4c64465cb888fbbb10fa293f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/8795b11a4c64465cb888fbbb10fa293f 2024-11-16T14:14:19,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/8795b11a4c64465cb888fbbb10fa293f, entries=150, sequenceid=99, filesize=11.7 K 2024-11-16T14:14:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/a8d7510ef01f41298504184a00922dee as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a8d7510ef01f41298504184a00922dee 2024-11-16T14:14:19,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a8d7510ef01f41298504184a00922dee, entries=150, sequenceid=99, filesize=11.7 K 2024-11-16T14:14:19,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~152.07 KB/155720, heapSize ~424.28 KB/434464, currentSize=33.54 KB/34350 for fd241767f9d916c6c1f6b781eb2b8ff7 in 639ms, sequenceid=99, compaction requested=true 2024-11-16T14:14:19,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:19,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:19,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:19,140 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:19,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:19,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:19,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:19,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T14:14:19,142 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:19,143 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/A is initiating minor compaction (all files) 2024-11-16T14:14:19,143 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/A in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:19,143 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/4dd357149d7c47998ac7f464b9a91899, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed579eab3d9149dbaaddbf612317985d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/366f065c0c514b9d8ccea2b384e6f799] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=33.0 K 2024-11-16T14:14:19,144 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dd357149d7c47998ac7f464b9a91899, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731766457356 2024-11-16T14:14:19,145 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ed579eab3d9149dbaaddbf612317985d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731766457572 2024-11-16T14:14:19,146 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 366f065c0c514b9d8ccea2b384e6f799, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731766458353 2024-11-16T14:14:19,178 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#A#compaction#18 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:19,179 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/f5839350f67a440b894e7a3385166ab9 is 50, key is test_row_0/A:col10/1731766458353/Put/seqid=0 2024-11-16T14:14:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:19,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:14:19,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:19,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:19,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:19,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:19,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:19,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:19,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/0a9e4019bcdd481aabb1e9f81ebf4979 is 50, key is test_row_0/A:col10/1731766458516/Put/seqid=0 2024-11-16T14:14:19,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741855_1031 (size=12207) 2024-11-16T14:14:19,228 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/f5839350f67a440b894e7a3385166ab9 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/f5839350f67a440b894e7a3385166ab9 2024-11-16T14:14:19,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741856_1032 (size=14341) 2024-11-16T14:14:19,246 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/A of fd241767f9d916c6c1f6b781eb2b8ff7 into f5839350f67a440b894e7a3385166ab9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:19,246 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:19,246 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/A, priority=13, startTime=1731766459139; duration=0sec 2024-11-16T14:14:19,246 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T14:14:19,246 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:19,246 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-16T14:14:19,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/0a9e4019bcdd481aabb1e9f81ebf4979 2024-11-16T14:14:19,252 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T14:14:19,252 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T14:14:19,254 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. because compaction request was cancelled 2024-11-16T14:14:19,254 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:19,257 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:19,261 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:19,262 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/B is initiating minor compaction (all files) 2024-11-16T14:14:19,262 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/B in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:19,262 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/0ac3cab406e54a508d25feecfa1660e5 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/0ac3cab406e54a508d25feecfa1660e5 2024-11-16T14:14:19,262 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7f1394528b544e90abf0d65c242aba45, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/a0b7109a7fd348b287ee976eb5c57922, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/8795b11a4c64465cb888fbbb10fa293f] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=33.0 K 2024-11-16T14:14:19,264 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f1394528b544e90abf0d65c242aba45, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731766457356 2024-11-16T14:14:19,265 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting a0b7109a7fd348b287ee976eb5c57922, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731766457572 2024-11-16T14:14:19,267 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8795b11a4c64465cb888fbbb10fa293f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731766458353 2024-11-16T14:14:19,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/a41f459a4cbe423c8dfa5fc7ec926762 is 50, key is test_row_0/B:col10/1731766458516/Put/seqid=0 2024-11-16T14:14:19,284 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/C of fd241767f9d916c6c1f6b781eb2b8ff7 into 0ac3cab406e54a508d25feecfa1660e5(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:19,285 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:19,285 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/C, priority=12, startTime=1731766458164; duration=0sec 2024-11-16T14:14:19,285 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:19,285 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:19,299 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:19,300 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/ef54d13a0eeb4ccbbab23b753644bd47 is 50, key is test_row_0/B:col10/1731766458353/Put/seqid=0 2024-11-16T14:14:19,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766459280, max timestamp=1731766459310Num uniques 859; 2024-11-16T14:14:19,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:19,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766459280, max timestamp=1731766459310Num uniques 859; 2024-11-16T14:14:19,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:19,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741857_1033 (size=12001) 2024-11-16T14:14:19,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/a41f459a4cbe423c8dfa5fc7ec926762 2024-11-16T14:14:19,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766459280, max timestamp=1731766459292Num uniques 859; 2024-11-16T14:14:19,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:19,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741858_1034 (size=12207) 2024-11-16T14:14:19,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/0867ac5532d8421390d17946103b1baa is 50, key is test_row_0/C:col10/1731766458516/Put/seqid=0 2024-11-16T14:14:19,362 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/ef54d13a0eeb4ccbbab23b753644bd47 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/ef54d13a0eeb4ccbbab23b753644bd47 2024-11-16T14:14:19,376 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/B of fd241767f9d916c6c1f6b781eb2b8ff7 into ef54d13a0eeb4ccbbab23b753644bd47(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:19,376 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:19,376 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/B, priority=13, startTime=1731766459140; duration=0sec 2024-11-16T14:14:19,376 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:19,376 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:19,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741859_1035 (size=12001) 2024-11-16T14:14:19,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766519386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766519388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766519397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766519507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766519507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766519516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766519566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766519567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766519716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766519716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766519725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:19,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/0867ac5532d8421390d17946103b1baa 2024-11-16T14:14:19,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/0a9e4019bcdd481aabb1e9f81ebf4979 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/0a9e4019bcdd481aabb1e9f81ebf4979 2024-11-16T14:14:19,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/0a9e4019bcdd481aabb1e9f81ebf4979, entries=200, sequenceid=111, filesize=14.0 K 2024-11-16T14:14:19,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/a41f459a4cbe423c8dfa5fc7ec926762 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/a41f459a4cbe423c8dfa5fc7ec926762 2024-11-16T14:14:19,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/a41f459a4cbe423c8dfa5fc7ec926762, entries=150, sequenceid=111, filesize=11.7 K 2024-11-16T14:14:19,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/0867ac5532d8421390d17946103b1baa as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/0867ac5532d8421390d17946103b1baa 2024-11-16T14:14:19,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/0867ac5532d8421390d17946103b1baa, entries=150, sequenceid=111, filesize=11.7 K 2024-11-16T14:14:19,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fd241767f9d916c6c1f6b781eb2b8ff7 in 679ms, sequenceid=111, compaction requested=true 2024-11-16T14:14:19,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:19,859 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T14:14:19,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:19,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:19,860 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T14:14:19,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:19,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:19,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:19,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:19,865 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T14:14:19,865 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T14:14:19,865 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T14:14:19,865 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. because compaction request was cancelled 2024-11-16T14:14:19,865 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T14:14:19,865 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:19,865 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. because compaction request was cancelled 2024-11-16T14:14:19,865 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:19,865 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:19,868 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:19,868 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/C is initiating minor compaction (all files) 2024-11-16T14:14:19,868 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/C in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:19,868 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/0ac3cab406e54a508d25feecfa1660e5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a8d7510ef01f41298504184a00922dee, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/0867ac5532d8421390d17946103b1baa] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=35.3 K 2024-11-16T14:14:19,870 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ac3cab406e54a508d25feecfa1660e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731766457356 2024-11-16T14:14:19,871 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8d7510ef01f41298504184a00922dee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731766458353 2024-11-16T14:14:19,871 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0867ac5532d8421390d17946103b1baa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731766458516 2024-11-16T14:14:19,894 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#C#compaction#23 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:19,895 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/e11853d53b56499284ff5ef8b1bc5fbb is 50, key is test_row_0/C:col10/1731766458516/Put/seqid=0 2024-11-16T14:14:19,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741860_1036 (size=12241) 2024-11-16T14:14:19,941 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/e11853d53b56499284ff5ef8b1bc5fbb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/e11853d53b56499284ff5ef8b1bc5fbb 2024-11-16T14:14:19,964 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/C of fd241767f9d916c6c1f6b781eb2b8ff7 into e11853d53b56499284ff5ef8b1bc5fbb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:19,964 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:19,964 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/C, priority=13, startTime=1731766459860; duration=0sec 2024-11-16T14:14:19,964 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:19,964 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:20,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-16T14:14:20,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:20,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T14:14:20,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:20,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T14:14:20,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:20,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T14:14:20,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/45295560950446d0952b9503d1ff5934 is 50, key is test_row_0/A:col10/1731766459394/Put/seqid=0 2024-11-16T14:14:20,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766520048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741861_1037 (size=12151) 2024-11-16T14:14:20,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/45295560950446d0952b9503d1ff5934 2024-11-16T14:14:20,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766520051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766520053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/b95d4680cda14247abb33a299a6aa76f is 50, key is test_row_0/B:col10/1731766459394/Put/seqid=0 2024-11-16T14:14:20,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741862_1038 (size=12151) 2024-11-16T14:14:20,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/b95d4680cda14247abb33a299a6aa76f 2024-11-16T14:14:20,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/30294b91dfa74d81b0c5a1fd0c172e1d is 50, key is test_row_0/C:col10/1731766459394/Put/seqid=0 2024-11-16T14:14:20,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741863_1039 (size=12151) 2024-11-16T14:14:20,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766520156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766520167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766520167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766520366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766520376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766520378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:14:20,446 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:20,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-16T14:14:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T14:14:20,451 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:20,452 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:20,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:20,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/30294b91dfa74d81b0c5a1fd0c172e1d 2024-11-16T14:14:20,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/45295560950446d0952b9503d1ff5934 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/45295560950446d0952b9503d1ff5934 2024-11-16T14:14:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T14:14:20,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/45295560950446d0952b9503d1ff5934, entries=150, sequenceid=153, filesize=11.9 K 2024-11-16T14:14:20,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/b95d4680cda14247abb33a299a6aa76f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/b95d4680cda14247abb33a299a6aa76f 2024-11-16T14:14:20,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/b95d4680cda14247abb33a299a6aa76f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-16T14:14:20,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/30294b91dfa74d81b0c5a1fd0c172e1d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/30294b91dfa74d81b0c5a1fd0c172e1d 2024-11-16T14:14:20,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/30294b91dfa74d81b0c5a1fd0c172e1d, entries=150, sequenceid=153, filesize=11.9 K 2024-11-16T14:14:20,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for fd241767f9d916c6c1f6b781eb2b8ff7 in 556ms, sequenceid=153, compaction requested=true 2024-11-16T14:14:20,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:20,586 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:20,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:20,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:20,587 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:20,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:20,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:20,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:20,591 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:20,591 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:20,591 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/A is initiating minor compaction (all files) 2024-11-16T14:14:20,591 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/B is initiating minor compaction (all files) 2024-11-16T14:14:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:20,591 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/A in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:20,591 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/B in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:20,591 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/ef54d13a0eeb4ccbbab23b753644bd47, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/a41f459a4cbe423c8dfa5fc7ec926762, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/b95d4680cda14247abb33a299a6aa76f] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=35.5 K 2024-11-16T14:14:20,591 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/f5839350f67a440b894e7a3385166ab9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/0a9e4019bcdd481aabb1e9f81ebf4979, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/45295560950446d0952b9503d1ff5934] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=37.8 K 2024-11-16T14:14:20,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:20,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:14:20,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:20,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:20,593 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ef54d13a0eeb4ccbbab23b753644bd47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731766458353 2024-11-16T14:14:20,593 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5839350f67a440b894e7a3385166ab9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731766458353 2024-11-16T14:14:20,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:20,593 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting a41f459a4cbe423c8dfa5fc7ec926762, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731766458516 2024-11-16T14:14:20,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:20,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:20,594 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a9e4019bcdd481aabb1e9f81ebf4979, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731766458516 2024-11-16T14:14:20,594 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting b95d4680cda14247abb33a299a6aa76f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731766459377 2024-11-16T14:14:20,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:20,594 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45295560950446d0952b9503d1ff5934, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731766459377 2024-11-16T14:14:20,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T14:14:20,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:20,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:20,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:20,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:20,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:20,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a1078b14a68647b4a2dcdb1bb1b42ce5 is 50, key is test_row_0/A:col10/1731766460589/Put/seqid=0 2024-11-16T14:14:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:20,621 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:20,622 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/2a822ebda6ee442cb5474a8d2b39cf37 is 50, key is test_row_0/B:col10/1731766459394/Put/seqid=0 2024-11-16T14:14:20,652 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#A#compaction#29 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:20,653 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/00fcd91098a04fc6962211c5847b5d2f is 50, key is test_row_0/A:col10/1731766459394/Put/seqid=0 2024-11-16T14:14:20,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741864_1040 (size=9757) 2024-11-16T14:14:20,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741865_1041 (size=12459) 2024-11-16T14:14:20,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741866_1042 (size=12459) 2024-11-16T14:14:20,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766460694, max timestamp=1731766460699Num uniques 859; 2024-11-16T14:14:20,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766460694, max timestamp=1731766460699Num uniques 859; 2024-11-16T14:14:20,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766460694, max timestamp=1731766460699Num uniques 859; 2024-11-16T14:14:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766520696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:20,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:20,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,719 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/00fcd91098a04fc6962211c5847b5d2f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/00fcd91098a04fc6962211c5847b5d2f 2024-11-16T14:14:20,732 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/A of fd241767f9d916c6c1f6b781eb2b8ff7 into 00fcd91098a04fc6962211c5847b5d2f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:20,732 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:20,732 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/A, priority=13, startTime=1731766460585; duration=0sec 2024-11-16T14:14:20,732 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:20,732 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:20,732 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T14:14:20,736 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T14:14:20,737 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T14:14:20,737 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. because compaction request was cancelled 2024-11-16T14:14:20,737 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:20,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766520746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766520746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766520748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766520748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T14:14:20,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:20,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:20,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:20,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:20,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T14:14:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:20,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766520815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766520856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766520866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766520866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766520866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:20,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T14:14:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:20,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766521025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a1078b14a68647b4a2dcdb1bb1b42ce5 2024-11-16T14:14:21,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766521066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T14:14:21,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:21,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:21,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/d0a10e9593a44de89f773ea81b454a81 is 50, key is test_row_0/B:col10/1731766460589/Put/seqid=0 2024-11-16T14:14:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:21,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T14:14:21,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766521076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766521077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766521086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,089 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/2a822ebda6ee442cb5474a8d2b39cf37 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/2a822ebda6ee442cb5474a8d2b39cf37 2024-11-16T14:14:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,100 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/B of fd241767f9d916c6c1f6b781eb2b8ff7 into 2a822ebda6ee442cb5474a8d2b39cf37(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:21,100 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:21,100 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/B, priority=13, startTime=1731766460587; duration=0sec 2024-11-16T14:14:21,100 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:21,100 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:21,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741867_1043 (size=9757) 2024-11-16T14:14:21,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/d0a10e9593a44de89f773ea81b454a81 2024-11-16T14:14:21,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/d9653ad20fd743f0865f502be606730a is 50, key is test_row_0/C:col10/1731766460589/Put/seqid=0 2024-11-16T14:14:21,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741868_1044 (size=9757) 2024-11-16T14:14:21,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T14:14:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:21,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766521346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766521375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T14:14:21,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:21,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:21,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:21,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766521396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766521397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766521397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T14:14:21,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:21,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/d9653ad20fd743f0865f502be606730a 2024-11-16T14:14:21,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:21,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:21,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:21,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a1078b14a68647b4a2dcdb1bb1b42ce5 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a1078b14a68647b4a2dcdb1bb1b42ce5 2024-11-16T14:14:21,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a1078b14a68647b4a2dcdb1bb1b42ce5, entries=100, sequenceid=164, filesize=9.5 K 2024-11-16T14:14:21,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/d0a10e9593a44de89f773ea81b454a81 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/d0a10e9593a44de89f773ea81b454a81 2024-11-16T14:14:21,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/d0a10e9593a44de89f773ea81b454a81, entries=100, sequenceid=164, filesize=9.5 K 2024-11-16T14:14:21,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/d9653ad20fd743f0865f502be606730a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d9653ad20fd743f0865f502be606730a 2024-11-16T14:14:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T14:14:21,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d9653ad20fd743f0865f502be606730a, entries=100, sequenceid=164, filesize=9.5 K 2024-11-16T14:14:21,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fd241767f9d916c6c1f6b781eb2b8ff7 in 1000ms, sequenceid=164, compaction requested=true 2024-11-16T14:14:21,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:21,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:21,593 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T14:14:21,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:21,593 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T14:14:21,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:21,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:21,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:21,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:21,594 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T14:14:21,594 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T14:14:21,594 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T14:14:21,594 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T14:14:21,594 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. because compaction request was cancelled 2024-11-16T14:14:21,594 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. because compaction request was cancelled 2024-11-16T14:14:21,594 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:21,594 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:21,594 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:21,599 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:21,599 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/C is initiating minor compaction (all files) 2024-11-16T14:14:21,599 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/C in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:21,599 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/e11853d53b56499284ff5ef8b1bc5fbb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/30294b91dfa74d81b0c5a1fd0c172e1d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d9653ad20fd743f0865f502be606730a] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=33.3 K 2024-11-16T14:14:21,600 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting e11853d53b56499284ff5ef8b1bc5fbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731766458516 2024-11-16T14:14:21,600 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 30294b91dfa74d81b0c5a1fd0c172e1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731766459377 2024-11-16T14:14:21,600 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting d9653ad20fd743f0865f502be606730a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731766460049 2024-11-16T14:14:21,618 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#C#compaction#32 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:21,619 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/904c1a6777ec44e5aa657d2e8858c7c2 is 50, key is test_row_0/C:col10/1731766460589/Put/seqid=0 2024-11-16T14:14:21,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741869_1045 (size=12493) 2024-11-16T14:14:21,652 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/904c1a6777ec44e5aa657d2e8858c7c2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/904c1a6777ec44e5aa657d2e8858c7c2 2024-11-16T14:14:21,666 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/C of fd241767f9d916c6c1f6b781eb2b8ff7 into 904c1a6777ec44e5aa657d2e8858c7c2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:21,666 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:21,666 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/C, priority=13, startTime=1731766461593; duration=0sec 2024-11-16T14:14:21,667 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:21,667 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:21,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T14:14:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:21,694 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T14:14:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:21,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:21,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:21,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:21,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:21,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:21,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d19f226072c3473f91fb5d1578c0c7b1 is 50, key is test_row_0/A:col10/1731766460732/Put/seqid=0 2024-11-16T14:14:21,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741870_1046 (size=12151) 2024-11-16T14:14:21,719 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d19f226072c3473f91fb5d1578c0c7b1 2024-11-16T14:14:21,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/636b4771bc0a45fca658b5062f4362b3 is 50, key is test_row_0/B:col10/1731766460732/Put/seqid=0 2024-11-16T14:14:21,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741871_1047 (size=12151) 2024-11-16T14:14:21,756 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/636b4771bc0a45fca658b5062f4362b3 2024-11-16T14:14:21,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/7059ff4d02d24aea91c9c1327ead885d is 50, key is test_row_0/C:col10/1731766460732/Put/seqid=0 2024-11-16T14:14:21,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741872_1048 (size=12151) 2024-11-16T14:14:21,802 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/7059ff4d02d24aea91c9c1327ead885d 2024-11-16T14:14:21,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d19f226072c3473f91fb5d1578c0c7b1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d19f226072c3473f91fb5d1578c0c7b1 2024-11-16T14:14:21,829 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d19f226072c3473f91fb5d1578c0c7b1, entries=150, sequenceid=203, filesize=11.9 K 2024-11-16T14:14:21,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/636b4771bc0a45fca658b5062f4362b3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/636b4771bc0a45fca658b5062f4362b3 2024-11-16T14:14:21,842 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/636b4771bc0a45fca658b5062f4362b3, entries=150, sequenceid=203, filesize=11.9 K 2024-11-16T14:14:21,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/7059ff4d02d24aea91c9c1327ead885d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7059ff4d02d24aea91c9c1327ead885d 2024-11-16T14:14:21,853 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7059ff4d02d24aea91c9c1327ead885d, entries=150, sequenceid=203, filesize=11.9 K 2024-11-16T14:14:21,854 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=0 B/0 for fd241767f9d916c6c1f6b781eb2b8ff7 in 160ms, sequenceid=203, compaction requested=true 2024-11-16T14:14:21,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:21,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:21,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-16T14:14:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-16T14:14:21,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-16T14:14:21,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-16T14:14:21,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.4180 sec 2024-11-16T14:14:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:21,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:14:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:21,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/5859f1fed38648b49546dc991abafbc0 is 50, key is test_row_0/A:col10/1731766461889/Put/seqid=0 2024-11-16T14:14:21,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741873_1049 (size=14541) 2024-11-16T14:14:21,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766461916, max timestamp=1731766461931Num uniques 859; 2024-11-16T14:14:21,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:14:21,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766461916, max timestamp=1731766461931Num uniques 859; 2024-11-16T14:14:21,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766461916, max timestamp=1731766461931Num uniques 859; 2024-11-16T14:14:21,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:14:21,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:14:21,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766521952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766521957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766521957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766521960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766521960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766522066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766522076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766522076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766522086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766522086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766522275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766522286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766522286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766522295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766522297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/5859f1fed38648b49546dc991abafbc0 2024-11-16T14:14:22,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/cd3a8e154a624bf1ae1bbd29c9975d3d is 50, key is test_row_0/B:col10/1731766461889/Put/seqid=0 2024-11-16T14:14:22,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741874_1050 (size=12151) 2024-11-16T14:14:22,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766522586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T14:14:22,597 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:22,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-16T14:14:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T14:14:22,600 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:22,601 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:22,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:22,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766522606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766522606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766522606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766522606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T14:14:22,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/cd3a8e154a624bf1ae1bbd29c9975d3d 2024-11-16T14:14:22,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T14:14:22,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:22,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:22,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/7c07f934f1794bdcb82a349d9765760f is 50, key is test_row_0/C:col10/1731766461889/Put/seqid=0 2024-11-16T14:14:22,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741875_1051 (size=12151) 2024-11-16T14:14:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T14:14:22,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T14:14:22,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:22,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:22,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:22,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:22,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:23,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T14:14:23,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:23,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:23,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:23,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:23,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:23,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766523096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:23,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:23,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766523115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:23,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766523116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:23,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766523126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:23,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766523127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:23,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:23,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/7c07f934f1794bdcb82a349d9765760f 2024-11-16T14:14:23,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/5859f1fed38648b49546dc991abafbc0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/5859f1fed38648b49546dc991abafbc0 2024-11-16T14:14:23,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/5859f1fed38648b49546dc991abafbc0, entries=200, sequenceid=213, filesize=14.2 K 2024-11-16T14:14:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T14:14:23,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/cd3a8e154a624bf1ae1bbd29c9975d3d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/cd3a8e154a624bf1ae1bbd29c9975d3d 2024-11-16T14:14:23,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T14:14:23,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:23,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:23,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:23,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:23,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:23,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/cd3a8e154a624bf1ae1bbd29c9975d3d, entries=150, sequenceid=213, filesize=11.9 K 2024-11-16T14:14:23,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/7c07f934f1794bdcb82a349d9765760f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7c07f934f1794bdcb82a349d9765760f 2024-11-16T14:14:23,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7c07f934f1794bdcb82a349d9765760f, entries=150, sequenceid=213, filesize=11.9 K 2024-11-16T14:14:23,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fd241767f9d916c6c1f6b781eb2b8ff7 in 1349ms, sequenceid=213, compaction requested=true 2024-11-16T14:14:23,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:23,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:23,247 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:14:23,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:23,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:23,247 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:14:23,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:23,249 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48908 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:14:23,249 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/A is initiating minor compaction (all files) 2024-11-16T14:14:23,249 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/A in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:23,250 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/00fcd91098a04fc6962211c5847b5d2f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a1078b14a68647b4a2dcdb1bb1b42ce5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d19f226072c3473f91fb5d1578c0c7b1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/5859f1fed38648b49546dc991abafbc0] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=47.8 K 2024-11-16T14:14:23,251 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:14:23,251 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/B is initiating minor compaction (all files) 2024-11-16T14:14:23,251 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/B in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:23,251 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/2a822ebda6ee442cb5474a8d2b39cf37, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/d0a10e9593a44de89f773ea81b454a81, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/636b4771bc0a45fca658b5062f4362b3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/cd3a8e154a624bf1ae1bbd29c9975d3d] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=45.4 K 2024-11-16T14:14:23,251 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00fcd91098a04fc6962211c5847b5d2f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731766459377 2024-11-16T14:14:23,252 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1078b14a68647b4a2dcdb1bb1b42ce5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731766460049 2024-11-16T14:14:23,252 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a822ebda6ee442cb5474a8d2b39cf37, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731766459377 2024-11-16T14:14:23,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:23,253 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d19f226072c3473f91fb5d1578c0c7b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766460732 2024-11-16T14:14:23,253 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a10e9593a44de89f773ea81b454a81, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731766460049 2024-11-16T14:14:23,254 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5859f1fed38648b49546dc991abafbc0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731766461884 2024-11-16T14:14:23,254 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 636b4771bc0a45fca658b5062f4362b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766460732 2024-11-16T14:14:23,255 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting cd3a8e154a624bf1ae1bbd29c9975d3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731766461889 2024-11-16T14:14:23,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:23,284 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:23,285 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/f677586147234d9aab4efc39db77d557 is 50, key is test_row_0/A:col10/1731766461889/Put/seqid=0 2024-11-16T14:14:23,286 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:23,287 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/5de909dd7dc84b0aba56d683c019af1a is 50, key is test_row_0/B:col10/1731766461889/Put/seqid=0 2024-11-16T14:14:23,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741877_1053 (size=12595) 2024-11-16T14:14:23,335 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/f677586147234d9aab4efc39db77d557 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/f677586147234d9aab4efc39db77d557 2024-11-16T14:14:23,346 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/A of fd241767f9d916c6c1f6b781eb2b8ff7 into f677586147234d9aab4efc39db77d557(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:23,346 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:23,346 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/A, priority=12, startTime=1731766463247; duration=0sec 2024-11-16T14:14:23,347 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:23,347 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:23,347 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:23,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741876_1052 (size=12595) 2024-11-16T14:14:23,352 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:23,352 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/C is initiating minor compaction (all files) 2024-11-16T14:14:23,352 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/C in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:23,353 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/904c1a6777ec44e5aa657d2e8858c7c2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7059ff4d02d24aea91c9c1327ead885d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7c07f934f1794bdcb82a349d9765760f] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=35.9 K 2024-11-16T14:14:23,354 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 904c1a6777ec44e5aa657d2e8858c7c2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731766459377 2024-11-16T14:14:23,356 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7059ff4d02d24aea91c9c1327ead885d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766460732 2024-11-16T14:14:23,358 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c07f934f1794bdcb82a349d9765760f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731766461889 2024-11-16T14:14:23,360 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/5de909dd7dc84b0aba56d683c019af1a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/5de909dd7dc84b0aba56d683c019af1a 2024-11-16T14:14:23,373 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/B of fd241767f9d916c6c1f6b781eb2b8ff7 into 5de909dd7dc84b0aba56d683c019af1a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:23,373 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:23,373 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/B, priority=12, startTime=1731766463247; duration=0sec 2024-11-16T14:14:23,374 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:23,374 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:23,377 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#C#compaction#41 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:23,378 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/1ec2ca37de4a41338627b3bec571a22b is 50, key is test_row_0/C:col10/1731766461889/Put/seqid=0 2024-11-16T14:14:23,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T14:14:23,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:23,384 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T14:14:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:23,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741878_1054 (size=12595) 2024-11-16T14:14:23,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/ed1f896bcff64267b858138d96de8dda is 50, key is test_row_0/A:col10/1731766461957/Put/seqid=0 2024-11-16T14:14:23,407 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/1ec2ca37de4a41338627b3bec571a22b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1ec2ca37de4a41338627b3bec571a22b 2024-11-16T14:14:23,417 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/C of fd241767f9d916c6c1f6b781eb2b8ff7 into 1ec2ca37de4a41338627b3bec571a22b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:23,418 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:23,418 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/C, priority=13, startTime=1731766463247; duration=0sec 2024-11-16T14:14:23,418 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:23,418 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:23,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741879_1055 (size=12151) 2024-11-16T14:14:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T14:14:23,829 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/ed1f896bcff64267b858138d96de8dda 2024-11-16T14:14:23,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/de77e3d9d3e14962863433f6135fc1ba is 50, key is test_row_0/B:col10/1731766461957/Put/seqid=0 2024-11-16T14:14:23,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741880_1056 (size=12151) 2024-11-16T14:14:23,848 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/de77e3d9d3e14962863433f6135fc1ba 2024-11-16T14:14:23,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/176c105b4de549e59bbd517fecc26b2e is 50, key is test_row_0/C:col10/1731766461957/Put/seqid=0 2024-11-16T14:14:23,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741881_1057 (size=12151) 2024-11-16T14:14:23,896 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/176c105b4de549e59bbd517fecc26b2e 2024-11-16T14:14:23,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/ed1f896bcff64267b858138d96de8dda as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed1f896bcff64267b858138d96de8dda 2024-11-16T14:14:23,920 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed1f896bcff64267b858138d96de8dda, entries=150, sequenceid=250, filesize=11.9 K 2024-11-16T14:14:23,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/de77e3d9d3e14962863433f6135fc1ba as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/de77e3d9d3e14962863433f6135fc1ba 2024-11-16T14:14:23,932 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/de77e3d9d3e14962863433f6135fc1ba, entries=150, sequenceid=250, filesize=11.9 K 2024-11-16T14:14:23,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/176c105b4de549e59bbd517fecc26b2e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/176c105b4de549e59bbd517fecc26b2e 2024-11-16T14:14:23,943 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/176c105b4de549e59bbd517fecc26b2e, entries=150, sequenceid=250, filesize=11.9 K 2024-11-16T14:14:23,945 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=0 B/0 for fd241767f9d916c6c1f6b781eb2b8ff7 in 562ms, sequenceid=250, compaction requested=false 2024-11-16T14:14:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-16T14:14:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-16T14:14:23,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-16T14:14:23,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3490 sec 2024-11-16T14:14:23,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3590 sec 2024-11-16T14:14:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:24,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:14:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:24,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/da5b4bc49bab4c27aa1850d40d2cbfee is 50, key is test_row_0/A:col10/1731766464136/Put/seqid=0 2024-11-16T14:14:24,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741882_1058 (size=14741) 2024-11-16T14:14:24,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766524177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766524178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731766464166, max timestamp=1731766464184Num uniques 859; 2024-11-16T14:14:24,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731766464166, max timestamp=1731766464184Num uniques 859; 2024-11-16T14:14:24,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-16T14:14:24,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731766464166, max timestamp=1731766464184Num uniques 859; 2024-11-16T14:14:24,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-16T14:14:24,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-16T14:14:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766524218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766524221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766524223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766524295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766524296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766524326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766524336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766524336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766524506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766524506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766524535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766524546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766524546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/da5b4bc49bab4c27aa1850d40d2cbfee 2024-11-16T14:14:24,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/60fbbd2a86c4445289f12b65c1c679cc is 50, key is test_row_0/B:col10/1731766464136/Put/seqid=0 2024-11-16T14:14:24,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741883_1059 (size=12301) 2024-11-16T14:14:24,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/60fbbd2a86c4445289f12b65c1c679cc 2024-11-16T14:14:24,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/610d8579c3414206bbaab4dde7da4a88 is 50, key is test_row_0/C:col10/1731766464136/Put/seqid=0 2024-11-16T14:14:24,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741884_1060 (size=12301) 2024-11-16T14:14:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T14:14:24,746 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:24,747 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-16T14:14:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T14:14:24,751 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:24,753 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:24,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766524818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766524826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766524845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T14:14:24,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766524856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766524856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:24,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:14:24,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:24,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:24,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:24,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:24,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:25,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:14:25,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:25,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:25,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:25,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:25,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T14:14:25,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/610d8579c3414206bbaab4dde7da4a88 2024-11-16T14:14:25,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/da5b4bc49bab4c27aa1850d40d2cbfee as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/da5b4bc49bab4c27aa1850d40d2cbfee 2024-11-16T14:14:25,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/da5b4bc49bab4c27aa1850d40d2cbfee, entries=200, sequenceid=262, filesize=14.4 K 2024-11-16T14:14:25,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/60fbbd2a86c4445289f12b65c1c679cc as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/60fbbd2a86c4445289f12b65c1c679cc 2024-11-16T14:14:25,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/60fbbd2a86c4445289f12b65c1c679cc, entries=150, sequenceid=262, filesize=12.0 K 2024-11-16T14:14:25,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/610d8579c3414206bbaab4dde7da4a88 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/610d8579c3414206bbaab4dde7da4a88 2024-11-16T14:14:25,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/610d8579c3414206bbaab4dde7da4a88, entries=150, sequenceid=262, filesize=12.0 K 2024-11-16T14:14:25,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for fd241767f9d916c6c1f6b781eb2b8ff7 in 975ms, sequenceid=262, compaction requested=true 2024-11-16T14:14:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:25,113 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:25,113 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:25,115 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39487 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:25,115 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/A is initiating minor compaction (all files) 2024-11-16T14:14:25,115 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/A in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:25,115 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/f677586147234d9aab4efc39db77d557, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed1f896bcff64267b858138d96de8dda, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/da5b4bc49bab4c27aa1850d40d2cbfee] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=38.6 K 2024-11-16T14:14:25,116 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f677586147234d9aab4efc39db77d557, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731766461889 2024-11-16T14:14:25,116 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:25,116 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/B is initiating minor compaction (all files) 2024-11-16T14:14:25,116 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/B in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:25,117 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/5de909dd7dc84b0aba56d683c019af1a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/de77e3d9d3e14962863433f6135fc1ba, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/60fbbd2a86c4445289f12b65c1c679cc] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=36.2 K 2024-11-16T14:14:25,117 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed1f896bcff64267b858138d96de8dda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731766461951 2024-11-16T14:14:25,117 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5de909dd7dc84b0aba56d683c019af1a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731766461889 2024-11-16T14:14:25,118 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting da5b4bc49bab4c27aa1850d40d2cbfee, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731766464131 2024-11-16T14:14:25,118 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting de77e3d9d3e14962863433f6135fc1ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731766461951 2024-11-16T14:14:25,119 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 60fbbd2a86c4445289f12b65c1c679cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731766464135 2024-11-16T14:14:25,138 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:25,139 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/ccac29a3b055450c8c6dd7e144f7e129 is 50, key is test_row_0/A:col10/1731766464136/Put/seqid=0 2024-11-16T14:14:25,142 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:25,143 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/70b19cfee85a487abf6484634664cf28 is 50, key is test_row_0/B:col10/1731766464136/Put/seqid=0 2024-11-16T14:14:25,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741886_1062 (size=12847) 2024-11-16T14:14:25,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741885_1061 (size=12847) 2024-11-16T14:14:25,182 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/70b19cfee85a487abf6484634664cf28 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/70b19cfee85a487abf6484634664cf28 2024-11-16T14:14:25,195 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/B of fd241767f9d916c6c1f6b781eb2b8ff7 into 70b19cfee85a487abf6484634664cf28(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:25,196 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:25,196 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/B, priority=13, startTime=1731766465113; duration=0sec 2024-11-16T14:14:25,196 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:25,196 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:25,196 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:25,197 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:25,198 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/C is initiating minor compaction (all files) 2024-11-16T14:14:25,198 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/C in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:25,198 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1ec2ca37de4a41338627b3bec571a22b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/176c105b4de549e59bbd517fecc26b2e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/610d8579c3414206bbaab4dde7da4a88] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=36.2 K 2024-11-16T14:14:25,200 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ec2ca37de4a41338627b3bec571a22b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731766461889 2024-11-16T14:14:25,202 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 176c105b4de549e59bbd517fecc26b2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731766461951 2024-11-16T14:14:25,203 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 610d8579c3414206bbaab4dde7da4a88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731766464135 2024-11-16T14:14:25,213 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#C#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:25,215 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/93e94397d63a4021a2bb3b2d3dca5ab8 is 50, key is test_row_0/C:col10/1731766464136/Put/seqid=0 2024-11-16T14:14:25,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:14:25,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:25,215 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-16T14:14:25,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:25,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:25,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:25,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:25,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:25,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:25,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741887_1063 (size=12847) 2024-11-16T14:14:25,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/535c28db97a645179ab03ad1880f9153 is 50, key is test_row_0/A:col10/1731766464217/Put/seqid=0 2024-11-16T14:14:25,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741888_1064 (size=12301) 2024-11-16T14:14:25,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:25,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766525355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:25,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766525356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:25,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766525366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:25,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766525366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766525366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T14:14:25,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:25,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766525466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766525465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:25,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:25,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:25,574 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/ccac29a3b055450c8c6dd7e144f7e129 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ccac29a3b055450c8c6dd7e144f7e129 2024-11-16T14:14:25,581 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/A of fd241767f9d916c6c1f6b781eb2b8ff7 into ccac29a3b055450c8c6dd7e144f7e129(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:25,581 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:25,581 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/A, priority=13, startTime=1731766465113; duration=0sec 2024-11-16T14:14:25,581 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:25,581 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:25,628 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/93e94397d63a4021a2bb3b2d3dca5ab8 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/93e94397d63a4021a2bb3b2d3dca5ab8 2024-11-16T14:14:25,633 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/535c28db97a645179ab03ad1880f9153 2024-11-16T14:14:25,635 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/C of fd241767f9d916c6c1f6b781eb2b8ff7 into 93e94397d63a4021a2bb3b2d3dca5ab8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:25,635 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:25,635 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/C, priority=13, startTime=1731766465113; duration=0sec 2024-11-16T14:14:25,635 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:25,635 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/1456f97e291b45848242d6bed42aa167 is 50, key is test_row_0/B:col10/1731766464217/Put/seqid=0 2024-11-16T14:14:25,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741889_1065 (size=12301) 2024-11-16T14:14:25,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766525685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:25,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:25,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766525686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:25,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:25,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T14:14:25,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766525995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766525995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,062 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/1456f97e291b45848242d6bed42aa167 2024-11-16T14:14:26,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/d455820eeb1e4c51afc492e3cf259b85 is 50, key is test_row_0/C:col10/1731766464217/Put/seqid=0 2024-11-16T14:14:26,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741890_1066 (size=12301) 2024-11-16T14:14:26,081 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/d455820eeb1e4c51afc492e3cf259b85 2024-11-16T14:14:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/535c28db97a645179ab03ad1880f9153 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/535c28db97a645179ab03ad1880f9153 2024-11-16T14:14:26,096 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/535c28db97a645179ab03ad1880f9153, entries=150, sequenceid=296, filesize=12.0 K 2024-11-16T14:14:26,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/1456f97e291b45848242d6bed42aa167 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/1456f97e291b45848242d6bed42aa167 2024-11-16T14:14:26,107 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/1456f97e291b45848242d6bed42aa167, entries=150, sequenceid=296, filesize=12.0 K 2024-11-16T14:14:26,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/d455820eeb1e4c51afc492e3cf259b85 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d455820eeb1e4c51afc492e3cf259b85 2024-11-16T14:14:26,117 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d455820eeb1e4c51afc492e3cf259b85, entries=150, sequenceid=296, filesize=12.0 K 2024-11-16T14:14:26,119 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for fd241767f9d916c6c1f6b781eb2b8ff7 in 904ms, sequenceid=296, compaction requested=false 2024-11-16T14:14:26,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:26,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:26,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-16T14:14:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-16T14:14:26,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-16T14:14:26,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3690 sec 2024-11-16T14:14:26,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3810 sec 2024-11-16T14:14:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:26,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:14:26,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:26,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:26,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:26,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:26,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:26,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:26,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a26bb1f903bf41d794b5ef21972d3c84 is 50, key is test_row_0/A:col10/1731766465334/Put/seqid=0 2024-11-16T14:14:26,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741891_1067 (size=17181) 2024-11-16T14:14:26,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a26bb1f903bf41d794b5ef21972d3c84 2024-11-16T14:14:26,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/919e697ee3774b35b1154995244150f7 is 50, key is test_row_0/B:col10/1731766465334/Put/seqid=0 2024-11-16T14:14:26,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766526425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766526427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766526429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741892_1068 (size=12301) 2024-11-16T14:14:26,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/919e697ee3774b35b1154995244150f7 2024-11-16T14:14:26,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/7d4de58d41a84a93958deb62e1fbcbcd is 50, key is test_row_0/C:col10/1731766465334/Put/seqid=0 2024-11-16T14:14:26,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741893_1069 (size=12301) 2024-11-16T14:14:26,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/7d4de58d41a84a93958deb62e1fbcbcd 2024-11-16T14:14:26,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a26bb1f903bf41d794b5ef21972d3c84 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a26bb1f903bf41d794b5ef21972d3c84 2024-11-16T14:14:26,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766526506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766526506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a26bb1f903bf41d794b5ef21972d3c84, entries=250, sequenceid=311, filesize=16.8 K 2024-11-16T14:14:26,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/919e697ee3774b35b1154995244150f7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/919e697ee3774b35b1154995244150f7 2024-11-16T14:14:26,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/919e697ee3774b35b1154995244150f7, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T14:14:26,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/7d4de58d41a84a93958deb62e1fbcbcd as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7d4de58d41a84a93958deb62e1fbcbcd 2024-11-16T14:14:26,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766526536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7d4de58d41a84a93958deb62e1fbcbcd, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T14:14:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for fd241767f9d916c6c1f6b781eb2b8ff7 in 159ms, sequenceid=311, compaction requested=true 2024-11-16T14:14:26,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:26,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:26,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:26,539 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:26,539 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:26,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:26,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:26,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:26,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:26,540 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:26,540 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/A is initiating minor compaction (all files) 2024-11-16T14:14:26,540 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/A in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:26,541 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ccac29a3b055450c8c6dd7e144f7e129, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/535c28db97a645179ab03ad1880f9153, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a26bb1f903bf41d794b5ef21972d3c84] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=41.3 K 2024-11-16T14:14:26,542 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccac29a3b055450c8c6dd7e144f7e129, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731766464135 2024-11-16T14:14:26,542 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:26,542 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/B is initiating minor compaction (all files) 2024-11-16T14:14:26,542 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/B in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:26,542 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/70b19cfee85a487abf6484634664cf28, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/1456f97e291b45848242d6bed42aa167, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/919e697ee3774b35b1154995244150f7] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=36.6 K 2024-11-16T14:14:26,543 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 70b19cfee85a487abf6484634664cf28, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731766464135 2024-11-16T14:14:26,543 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 535c28db97a645179ab03ad1880f9153, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731766464203 2024-11-16T14:14:26,543 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1456f97e291b45848242d6bed42aa167, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731766464203 2024-11-16T14:14:26,544 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a26bb1f903bf41d794b5ef21972d3c84, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731766465334 2024-11-16T14:14:26,544 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 919e697ee3774b35b1154995244150f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731766465334 2024-11-16T14:14:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:26,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:14:26,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:26,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:26,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:26,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:26,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:26,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:26,571 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:26,572 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/cac012d7fdd64c8e8c9c349878b230d4 is 50, key is test_row_0/B:col10/1731766465334/Put/seqid=0 2024-11-16T14:14:26,579 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:26,579 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/e783eed8adda437aa5ad9504cc752098 is 50, key is test_row_0/A:col10/1731766465334/Put/seqid=0 2024-11-16T14:14:26,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/82dc7bb6673c4a45832afd60461a5dbb is 50, key is test_row_0/A:col10/1731766466549/Put/seqid=0 2024-11-16T14:14:26,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766526584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766526586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741894_1070 (size=12949) 2024-11-16T14:14:26,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741896_1072 (size=14741) 2024-11-16T14:14:26,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741895_1071 (size=12949) 2024-11-16T14:14:26,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766526695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766526696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766526745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T14:14:26,896 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:26,896 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-16T14:14:26,899 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T14:14:26,900 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:26,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:26,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766526907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766526907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T14:14:27,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/82dc7bb6673c4a45832afd60461a5dbb 2024-11-16T14:14:27,013 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/cac012d7fdd64c8e8c9c349878b230d4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/cac012d7fdd64c8e8c9c349878b230d4 2024-11-16T14:14:27,021 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/B of fd241767f9d916c6c1f6b781eb2b8ff7 into cac012d7fdd64c8e8c9c349878b230d4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:27,021 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:27,021 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/B, priority=13, startTime=1731766466539; duration=0sec 2024-11-16T14:14:27,022 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:27,022 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:27,022 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:27,022 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:27,023 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/C is initiating minor compaction (all files) 2024-11-16T14:14:27,023 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/C in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:27,023 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/93e94397d63a4021a2bb3b2d3dca5ab8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d455820eeb1e4c51afc492e3cf259b85, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7d4de58d41a84a93958deb62e1fbcbcd] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=36.6 K 2024-11-16T14:14:27,023 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 93e94397d63a4021a2bb3b2d3dca5ab8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731766464135 2024-11-16T14:14:27,024 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting d455820eeb1e4c51afc492e3cf259b85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731766464203 2024-11-16T14:14:27,024 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d4de58d41a84a93958deb62e1fbcbcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731766465334 2024-11-16T14:14:27,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/ecf0192daf0f44e1aeb6faacba88adf8 is 50, key is test_row_0/B:col10/1731766466549/Put/seqid=0 2024-11-16T14:14:27,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741897_1073 (size=12301) 2024-11-16T14:14:27,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/ecf0192daf0f44e1aeb6faacba88adf8 2024-11-16T14:14:27,036 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/e783eed8adda437aa5ad9504cc752098 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/e783eed8adda437aa5ad9504cc752098 2024-11-16T14:14:27,037 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:27,037 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/c65ef5dd0ec645aa80f365eb8fc99cd0 is 50, key is test_row_0/C:col10/1731766465334/Put/seqid=0 2024-11-16T14:14:27,049 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/A of fd241767f9d916c6c1f6b781eb2b8ff7 into e783eed8adda437aa5ad9504cc752098(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:27,049 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:27,049 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/A, priority=13, startTime=1731766466539; duration=0sec 2024-11-16T14:14:27,050 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:27,050 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:27,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/d793b8551f994c5fb19c474be3b556a4 is 50, key is test_row_0/C:col10/1731766466549/Put/seqid=0 2024-11-16T14:14:27,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T14:14:27,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:27,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:27,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:27,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:27,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:27,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766527056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:27,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741898_1074 (size=12949) 2024-11-16T14:14:27,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741899_1075 (size=12301) 2024-11-16T14:14:27,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/d793b8551f994c5fb19c474be3b556a4 2024-11-16T14:14:27,082 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/c65ef5dd0ec645aa80f365eb8fc99cd0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c65ef5dd0ec645aa80f365eb8fc99cd0 2024-11-16T14:14:27,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/82dc7bb6673c4a45832afd60461a5dbb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/82dc7bb6673c4a45832afd60461a5dbb 2024-11-16T14:14:27,090 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/C of fd241767f9d916c6c1f6b781eb2b8ff7 into c65ef5dd0ec645aa80f365eb8fc99cd0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:27,090 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:27,090 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/C, priority=13, startTime=1731766466539; duration=0sec 2024-11-16T14:14:27,090 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:27,090 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:27,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/82dc7bb6673c4a45832afd60461a5dbb, entries=200, sequenceid=331, filesize=14.4 K 2024-11-16T14:14:27,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/ecf0192daf0f44e1aeb6faacba88adf8 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/ecf0192daf0f44e1aeb6faacba88adf8 2024-11-16T14:14:27,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/ecf0192daf0f44e1aeb6faacba88adf8, entries=150, sequenceid=331, filesize=12.0 K 2024-11-16T14:14:27,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/d793b8551f994c5fb19c474be3b556a4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d793b8551f994c5fb19c474be3b556a4 2024-11-16T14:14:27,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d793b8551f994c5fb19c474be3b556a4, entries=150, sequenceid=331, filesize=12.0 K 2024-11-16T14:14:27,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for fd241767f9d916c6c1f6b781eb2b8ff7 in 566ms, sequenceid=331, compaction requested=false 2024-11-16T14:14:27,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:27,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T14:14:27,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:27,206 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:14:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:27,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/6e4c597aafa9495b8cdecc9b3a31da5b is 50, key is test_row_0/A:col10/1731766466578/Put/seqid=0 2024-11-16T14:14:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T14:14:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:27,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:27,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741900_1076 (size=12301) 2024-11-16T14:14:27,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766527262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:27,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766527263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:27,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766527376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766527376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:27,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766527516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:27,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766527516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:27,524 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in fd241767f9d916c6c1f6b781eb2b8ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:27,524 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in fd241767f9d916c6c1f6b781eb2b8ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T14:14:27,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766527566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:27,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:27,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766527585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:27,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766527585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:27,624 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/6e4c597aafa9495b8cdecc9b3a31da5b 2024-11-16T14:14:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/76d44a752305481ea9efd56f98638cf7 is 50, key is test_row_0/B:col10/1731766466578/Put/seqid=0 2024-11-16T14:14:27,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741901_1077 (size=12301) 2024-11-16T14:14:27,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766527896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766527896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:27,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T14:14:28,053 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/76d44a752305481ea9efd56f98638cf7 2024-11-16T14:14:28,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/71c379258d244e94909434591d66a04f is 50, key is test_row_0/C:col10/1731766466578/Put/seqid=0 2024-11-16T14:14:28,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741902_1078 (size=12301) 2024-11-16T14:14:28,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766528405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:28,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766528406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:28,466 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/71c379258d244e94909434591d66a04f 2024-11-16T14:14:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/6e4c597aafa9495b8cdecc9b3a31da5b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/6e4c597aafa9495b8cdecc9b3a31da5b 2024-11-16T14:14:28,477 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/6e4c597aafa9495b8cdecc9b3a31da5b, entries=150, sequenceid=346, filesize=12.0 K 2024-11-16T14:14:28,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/76d44a752305481ea9efd56f98638cf7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/76d44a752305481ea9efd56f98638cf7 2024-11-16T14:14:28,485 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/76d44a752305481ea9efd56f98638cf7, entries=150, sequenceid=346, filesize=12.0 K 2024-11-16T14:14:28,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/71c379258d244e94909434591d66a04f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/71c379258d244e94909434591d66a04f 2024-11-16T14:14:28,492 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/71c379258d244e94909434591d66a04f, entries=150, sequenceid=346, filesize=12.0 K 2024-11-16T14:14:28,496 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fd241767f9d916c6c1f6b781eb2b8ff7 in 1289ms, sequenceid=346, compaction requested=true 2024-11-16T14:14:28,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:28,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:28,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-16T14:14:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-16T14:14:28,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-16T14:14:28,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5970 sec 2024-11-16T14:14:28,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6040 sec 2024-11-16T14:14:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:28,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T14:14:28,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766467246, max timestamp=1731766468586Num uniques 859; 2024-11-16T14:14:28,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766467246, max timestamp=1731766468586Num uniques 859; 2024-11-16T14:14:28,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:28,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766467246, max timestamp=1731766468586Num uniques 859; 2024-11-16T14:14:28,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:28,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:28,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:28,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:28,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:28,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:28,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:28,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:28,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/cf75286ff9a247c09126468277e2b8df is 50, key is test_row_0/A:col10/1731766467261/Put/seqid=0 2024-11-16T14:14:28,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741903_1079 (size=14741) 2024-11-16T14:14:28,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766468637, max timestamp=1731766468688Num uniques 859; 2024-11-16T14:14:28,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766468637, max timestamp=1731766468688Num uniques 859; 2024-11-16T14:14:28,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:14:28,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:14:28,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766468637, max timestamp=1731766468688Num uniques 859; 2024-11-16T14:14:28,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:14:28,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731766468719, max timestamp=1731766468788Num uniques 1068; 2024-11-16T14:14:28,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-16T14:14:28,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731766468719, max timestamp=1731766468788Num uniques 1068; 2024-11-16T14:14:28,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731766468719, max timestamp=1731766468788Num uniques 1068; 2024-11-16T14:14:28,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-16T14:14:28,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-16T14:14:28,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731766468842, max timestamp=1731766468878Num uniques 1077; 2024-11-16T14:14:28,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-16T14:14:28,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731766468842, max timestamp=1731766468878Num uniques 1077; 2024-11-16T14:14:28,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-16T14:14:28,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731766468842, max timestamp=1731766468878Num uniques 1077; 2024-11-16T14:14:28,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-16T14:14:28,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731766468949, max timestamp=1731766468968Num uniques 1036; 2024-11-16T14:14:28,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731766468949, max timestamp=1731766468968Num uniques 1036; 2024-11-16T14:14:28,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-16T14:14:28,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-16T14:14:28,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731766468949, max timestamp=1731766468968Num uniques 1036; 2024-11-16T14:14:28,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-16T14:14:29,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/cf75286ff9a247c09126468277e2b8df 2024-11-16T14:14:29,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/3af381cb626e455ba82ddb3dfc9454a9 is 50, key is test_row_0/B:col10/1731766467261/Put/seqid=0 2024-11-16T14:14:29,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741904_1080 (size=12301) 2024-11-16T14:14:29,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/3af381cb626e455ba82ddb3dfc9454a9 2024-11-16T14:14:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T14:14:29,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/aa341c0f35c3412ea558005c2a6e283b is 50, key is test_row_0/C:col10/1731766467261/Put/seqid=0 2024-11-16T14:14:29,046 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:29,046 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-16T14:14:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T14:14:29,049 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:29,051 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:29,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:29,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741905_1081 (size=12301) 2024-11-16T14:14:29,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/aa341c0f35c3412ea558005c2a6e283b 2024-11-16T14:14:29,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/cf75286ff9a247c09126468277e2b8df as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/cf75286ff9a247c09126468277e2b8df 2024-11-16T14:14:29,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/cf75286ff9a247c09126468277e2b8df, entries=200, sequenceid=368, filesize=14.4 K 2024-11-16T14:14:29,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/3af381cb626e455ba82ddb3dfc9454a9 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/3af381cb626e455ba82ddb3dfc9454a9 2024-11-16T14:14:29,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731766468989, max timestamp=1731766469074Num uniques 1095; 2024-11-16T14:14:29,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731766468989, max timestamp=1731766469074Num uniques 1095; 2024-11-16T14:14:29,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-16T14:14:29,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-16T14:14:29,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731766468989, max timestamp=1731766469074Num uniques 1095; 2024-11-16T14:14:29,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-16T14:14:29,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/3af381cb626e455ba82ddb3dfc9454a9, entries=150, sequenceid=368, filesize=12.0 K 2024-11-16T14:14:29,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/aa341c0f35c3412ea558005c2a6e283b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/aa341c0f35c3412ea558005c2a6e283b 2024-11-16T14:14:29,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/aa341c0f35c3412ea558005c2a6e283b, entries=150, sequenceid=368, filesize=12.0 K 2024-11-16T14:14:29,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=67.09 KB/68700 for fd241767f9d916c6c1f6b781eb2b8ff7 in 513ms, sequenceid=368, compaction requested=true 2024-11-16T14:14:29,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:29,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:29,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:29,102 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:14:29,102 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:14:29,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:29,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:29,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:29,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:29,105 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:14:29,105 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/B is initiating minor compaction (all files) 2024-11-16T14:14:29,106 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/B in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:29,106 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/cac012d7fdd64c8e8c9c349878b230d4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/ecf0192daf0f44e1aeb6faacba88adf8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/76d44a752305481ea9efd56f98638cf7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/3af381cb626e455ba82ddb3dfc9454a9] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=48.7 K 2024-11-16T14:14:29,107 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting cac012d7fdd64c8e8c9c349878b230d4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731766465334 2024-11-16T14:14:29,108 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ecf0192daf0f44e1aeb6faacba88adf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731766466404 2024-11-16T14:14:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:29,108 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:14:29,109 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/A is initiating minor compaction (all files) 2024-11-16T14:14:29,109 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/A in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:29,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=73.80 KB heapSize=197.63 KB 2024-11-16T14:14:29,109 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/e783eed8adda437aa5ad9504cc752098, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/82dc7bb6673c4a45832afd60461a5dbb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/6e4c597aafa9495b8cdecc9b3a31da5b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/cf75286ff9a247c09126468277e2b8df] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=53.4 K 2024-11-16T14:14:29,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:29,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:29,109 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 76d44a752305481ea9efd56f98638cf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731766466572 2024-11-16T14:14:29,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:29,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:29,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:29,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:29,110 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e783eed8adda437aa5ad9504cc752098, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731766465334 2024-11-16T14:14:29,111 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af381cb626e455ba82ddb3dfc9454a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731766467246 2024-11-16T14:14:29,111 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82dc7bb6673c4a45832afd60461a5dbb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731766466404 2024-11-16T14:14:29,112 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e4c597aafa9495b8cdecc9b3a31da5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731766466572 2024-11-16T14:14:29,113 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf75286ff9a247c09126468277e2b8df, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731766467246 2024-11-16T14:14:29,147 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:29,148 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:29,149 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/db5ea9e4f6474cc79ebe6fd5be81ea37 is 50, key is test_row_0/B:col10/1731766467261/Put/seqid=0 2024-11-16T14:14:29,150 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a453b70530894d33b91e1d54a5558fe3 is 50, key is test_row_0/A:col10/1731766467261/Put/seqid=0 2024-11-16T14:14:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T14:14:29,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/1b0b61bce0644baf8bc22fc72f867f14 is 50, key is test_row_0/A:col10/1731766469091/Put/seqid=0 2024-11-16T14:14:29,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741907_1083 (size=13085) 2024-11-16T14:14:29,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741906_1082 (size=13085) 2024-11-16T14:14:29,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741908_1084 (size=12301) 2024-11-16T14:14:29,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/1b0b61bce0644baf8bc22fc72f867f14 2024-11-16T14:14:29,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T14:14:29,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:29,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:29,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:29,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:29,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:29,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/e470a4a36eb84d0dafa67cd7e520e731 is 50, key is test_row_0/B:col10/1731766469091/Put/seqid=0 2024-11-16T14:14:29,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766529223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:29,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741909_1085 (size=12301) 2024-11-16T14:14:29,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/e470a4a36eb84d0dafa67cd7e520e731 2024-11-16T14:14:29,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/7dea3772ca694b678f123c04a5534c8e is 50, key is test_row_0/C:col10/1731766469091/Put/seqid=0 2024-11-16T14:14:29,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741910_1086 (size=12301) 2024-11-16T14:14:29,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766529335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:29,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T14:14:29,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:29,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:29,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:29,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:29,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T14:14:29,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766529415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:29,417 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in fd241767f9d916c6c1f6b781eb2b8ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:29,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766529426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:29,434 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in fd241767f9d916c6c1f6b781eb2b8ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:29,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T14:14:29,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:29,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:29,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:29,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:29,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766529536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:29,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766529536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:29,544 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in fd241767f9d916c6c1f6b781eb2b8ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:29,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:29,544 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in fd241767f9d916c6c1f6b781eb2b8ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:29,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766529546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:29,579 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a453b70530894d33b91e1d54a5558fe3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a453b70530894d33b91e1d54a5558fe3 2024-11-16T14:14:29,584 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/A of fd241767f9d916c6c1f6b781eb2b8ff7 into a453b70530894d33b91e1d54a5558fe3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:29,584 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:29,584 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/A, priority=12, startTime=1731766469102; duration=0sec 2024-11-16T14:14:29,585 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:29,585 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:29,585 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:14:29,590 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:14:29,591 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/C is initiating minor compaction (all files) 2024-11-16T14:14:29,591 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/C in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:29,591 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c65ef5dd0ec645aa80f365eb8fc99cd0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d793b8551f994c5fb19c474be3b556a4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/71c379258d244e94909434591d66a04f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/aa341c0f35c3412ea558005c2a6e283b] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=48.7 K 2024-11-16T14:14:29,592 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c65ef5dd0ec645aa80f365eb8fc99cd0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731766465334 2024-11-16T14:14:29,593 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d793b8551f994c5fb19c474be3b556a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731766466404 2024-11-16T14:14:29,593 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/db5ea9e4f6474cc79ebe6fd5be81ea37 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/db5ea9e4f6474cc79ebe6fd5be81ea37 2024-11-16T14:14:29,593 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c379258d244e94909434591d66a04f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731766466572 2024-11-16T14:14:29,594 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa341c0f35c3412ea558005c2a6e283b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731766467246 2024-11-16T14:14:29,600 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/B of fd241767f9d916c6c1f6b781eb2b8ff7 into db5ea9e4f6474cc79ebe6fd5be81ea37(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:29,600 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:29,601 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/B, priority=12, startTime=1731766469102; duration=0sec 2024-11-16T14:14:29,601 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:29,601 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:29,606 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:29,607 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/557ca50786bc4d96a44dab2505671b38 is 50, key is test_row_0/C:col10/1731766467261/Put/seqid=0 2024-11-16T14:14:29,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741911_1087 (size=13085) 2024-11-16T14:14:29,621 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/557ca50786bc4d96a44dab2505671b38 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/557ca50786bc4d96a44dab2505671b38 2024-11-16T14:14:29,630 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/C of fd241767f9d916c6c1f6b781eb2b8ff7 into 557ca50786bc4d96a44dab2505671b38(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:29,630 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:29,630 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/C, priority=12, startTime=1731766469103; duration=0sec 2024-11-16T14:14:29,630 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:29,630 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:29,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T14:14:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:29,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T14:14:29,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/7dea3772ca694b678f123c04a5534c8e 2024-11-16T14:14:29,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/1b0b61bce0644baf8bc22fc72f867f14 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/1b0b61bce0644baf8bc22fc72f867f14 2024-11-16T14:14:29,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/1b0b61bce0644baf8bc22fc72f867f14, entries=150, sequenceid=463, filesize=12.0 K 2024-11-16T14:14:29,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/e470a4a36eb84d0dafa67cd7e520e731 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/e470a4a36eb84d0dafa67cd7e520e731 2024-11-16T14:14:29,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/e470a4a36eb84d0dafa67cd7e520e731, entries=150, sequenceid=463, filesize=12.0 K 2024-11-16T14:14:29,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/7dea3772ca694b678f123c04a5534c8e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7dea3772ca694b678f123c04a5534c8e 2024-11-16T14:14:29,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7dea3772ca694b678f123c04a5534c8e, entries=150, sequenceid=463, filesize=12.0 K 2024-11-16T14:14:29,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~197.58 KB/202320, currentSize=107.34 KB/109920 for fd241767f9d916c6c1f6b781eb2b8ff7 in 603ms, sequenceid=463, compaction requested=false 2024-11-16T14:14:29,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:29,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T14:14:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:29,817 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:14:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:29,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d32d5d7b58234543885b2e439f0fdb55 is 50, key is test_row_0/A:col10/1731766469198/Put/seqid=0 2024-11-16T14:14:29,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741912_1088 (size=12301) 2024-11-16T14:14:29,838 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d32d5d7b58234543885b2e439f0fdb55 2024-11-16T14:14:29,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/8b2a2b7bd1c746e387ff78c92c86d6d7 is 50, key is test_row_0/B:col10/1731766469198/Put/seqid=0 2024-11-16T14:14:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:29,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:29,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741913_1089 (size=12301) 2024-11-16T14:14:29,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766529898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:30,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766530005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T14:14:30,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766530215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:30,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:30,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:30,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:30,277 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/8b2a2b7bd1c746e387ff78c92c86d6d7 2024-11-16T14:14:30,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/f618164f43964365b0c51e3c759d7c23 is 50, key is test_row_0/C:col10/1731766469198/Put/seqid=0 2024-11-16T14:14:30,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741914_1090 (size=12301) 2024-11-16T14:14:30,305 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/f618164f43964365b0c51e3c759d7c23 2024-11-16T14:14:30,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d32d5d7b58234543885b2e439f0fdb55 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d32d5d7b58234543885b2e439f0fdb55 2024-11-16T14:14:30,317 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d32d5d7b58234543885b2e439f0fdb55, entries=150, sequenceid=485, filesize=12.0 K 2024-11-16T14:14:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/8b2a2b7bd1c746e387ff78c92c86d6d7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/8b2a2b7bd1c746e387ff78c92c86d6d7 2024-11-16T14:14:30,323 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/8b2a2b7bd1c746e387ff78c92c86d6d7, entries=150, sequenceid=485, filesize=12.0 K 2024-11-16T14:14:30,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/f618164f43964365b0c51e3c759d7c23 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/f618164f43964365b0c51e3c759d7c23 2024-11-16T14:14:30,333 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/f618164f43964365b0c51e3c759d7c23, entries=150, sequenceid=485, filesize=12.0 K 2024-11-16T14:14:30,334 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for fd241767f9d916c6c1f6b781eb2b8ff7 in 517ms, sequenceid=485, compaction requested=true 2024-11-16T14:14:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:30,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-16T14:14:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-16T14:14:30,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-16T14:14:30,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2860 sec 2024-11-16T14:14:30,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2940 sec 2024-11-16T14:14:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:30,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T14:14:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:30,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:30,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:30,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/0611ef8e0354499c96a1499d947dadc7 is 50, key is test_row_0/A:col10/1731766470526/Put/seqid=0 2024-11-16T14:14:30,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741915_1091 (size=14741) 2024-11-16T14:14:30,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766530598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:30,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766530705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:30,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766530916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:30,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/0611ef8e0354499c96a1499d947dadc7 2024-11-16T14:14:30,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/76317b3e7b904743b164ae4859d59c25 is 50, key is test_row_0/B:col10/1731766470526/Put/seqid=0 2024-11-16T14:14:30,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741916_1092 (size=12301) 2024-11-16T14:14:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T14:14:31,196 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:31,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-16T14:14:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T14:14:31,198 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:31,199 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:31,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:31,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766531225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T14:14:31,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T14:14:31,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:31,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:31,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:31,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:31,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:31,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/76317b3e7b904743b164ae4859d59c25 2024-11-16T14:14:31,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/9f0ab4c837804cbb88f8aaebabb8129a is 50, key is test_row_0/C:col10/1731766470526/Put/seqid=0 2024-11-16T14:14:31,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741917_1093 (size=12301) 2024-11-16T14:14:31,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766531436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:31,437 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in fd241767f9d916c6c1f6b781eb2b8ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:31,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766531455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:31,456 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in fd241767f9d916c6c1f6b781eb2b8ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:31,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:31,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:31,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:31,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T14:14:31,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:31,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:31,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:31,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:31,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T14:14:31,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T14:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:31,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:31,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:31,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766531735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:31,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/9f0ab4c837804cbb88f8aaebabb8129a 2024-11-16T14:14:31,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/0611ef8e0354499c96a1499d947dadc7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/0611ef8e0354499c96a1499d947dadc7 2024-11-16T14:14:31,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/0611ef8e0354499c96a1499d947dadc7, entries=200, sequenceid=499, filesize=14.4 K 2024-11-16T14:14:31,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/76317b3e7b904743b164ae4859d59c25 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/76317b3e7b904743b164ae4859d59c25 2024-11-16T14:14:31,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/76317b3e7b904743b164ae4859d59c25, entries=150, sequenceid=499, filesize=12.0 K 2024-11-16T14:14:31,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/9f0ab4c837804cbb88f8aaebabb8129a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/9f0ab4c837804cbb88f8aaebabb8129a 2024-11-16T14:14:31,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/9f0ab4c837804cbb88f8aaebabb8129a, entries=150, sequenceid=499, filesize=12.0 K 2024-11-16T14:14:31,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fd241767f9d916c6c1f6b781eb2b8ff7 in 1272ms, sequenceid=499, compaction requested=true 2024-11-16T14:14:31,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:31,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:31,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:31,800 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:14:31,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:31,800 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:14:31,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:31,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:31,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:31,802 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52428 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:14:31,802 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/A is initiating minor compaction (all files) 2024-11-16T14:14:31,802 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/A in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:31,802 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a453b70530894d33b91e1d54a5558fe3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/1b0b61bce0644baf8bc22fc72f867f14, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d32d5d7b58234543885b2e439f0fdb55, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/0611ef8e0354499c96a1499d947dadc7] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=51.2 K 2024-11-16T14:14:31,803 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:14:31,803 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a453b70530894d33b91e1d54a5558fe3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731766467246 2024-11-16T14:14:31,803 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/B is initiating minor compaction (all files) 2024-11-16T14:14:31,803 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/B in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:31,803 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b0b61bce0644baf8bc22fc72f867f14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731766469091 2024-11-16T14:14:31,803 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/db5ea9e4f6474cc79ebe6fd5be81ea37, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/e470a4a36eb84d0dafa67cd7e520e731, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/8b2a2b7bd1c746e387ff78c92c86d6d7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/76317b3e7b904743b164ae4859d59c25] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=48.8 K 2024-11-16T14:14:31,803 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32d5d7b58234543885b2e439f0fdb55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731766469198 2024-11-16T14:14:31,804 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting db5ea9e4f6474cc79ebe6fd5be81ea37, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731766467246 2024-11-16T14:14:31,804 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting e470a4a36eb84d0dafa67cd7e520e731, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731766469091 2024-11-16T14:14:31,804 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0611ef8e0354499c96a1499d947dadc7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731766469877 2024-11-16T14:14:31,804 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b2a2b7bd1c746e387ff78c92c86d6d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731766469198 2024-11-16T14:14:31,805 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 76317b3e7b904743b164ae4859d59c25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731766469881 2024-11-16T14:14:31,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T14:14:31,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:31,813 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T14:14:31,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:31,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:31,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:31,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:31,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:31,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:31,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/50d0c530a6e542d790721433b70c5ba5 is 50, key is test_row_0/A:col10/1731766470570/Put/seqid=0 2024-11-16T14:14:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T14:14:31,828 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#B#compaction#82 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:31,829 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/c4b468e7bef54e84959bcc6eaa07b951 is 50, key is test_row_0/B:col10/1731766470526/Put/seqid=0 2024-11-16T14:14:31,831 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#A#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:31,832 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/115e123d65344f8584951dca94dccd4f is 50, key is test_row_0/A:col10/1731766470526/Put/seqid=0 2024-11-16T14:14:31,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741918_1094 (size=12301) 2024-11-16T14:14:31,839 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/50d0c530a6e542d790721433b70c5ba5 2024-11-16T14:14:31,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741919_1095 (size=13221) 2024-11-16T14:14:31,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/dacb5f9cf8ed499186a0e29c1dabf097 is 50, key is test_row_0/B:col10/1731766470570/Put/seqid=0 2024-11-16T14:14:31,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741920_1096 (size=13221) 2024-11-16T14:14:31,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741921_1097 (size=12301) 2024-11-16T14:14:31,857 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/c4b468e7bef54e84959bcc6eaa07b951 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/c4b468e7bef54e84959bcc6eaa07b951 2024-11-16T14:14:31,862 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/115e123d65344f8584951dca94dccd4f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/115e123d65344f8584951dca94dccd4f 2024-11-16T14:14:31,864 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/B of fd241767f9d916c6c1f6b781eb2b8ff7 into c4b468e7bef54e84959bcc6eaa07b951(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:31,865 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:31,865 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/B, priority=12, startTime=1731766471800; duration=0sec 2024-11-16T14:14:31,866 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:31,866 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:31,866 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:14:31,868 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:14:31,868 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/C is initiating minor compaction (all files) 2024-11-16T14:14:31,869 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/C in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:31,869 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/557ca50786bc4d96a44dab2505671b38, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7dea3772ca694b678f123c04a5534c8e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/f618164f43964365b0c51e3c759d7c23, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/9f0ab4c837804cbb88f8aaebabb8129a] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=48.8 K 2024-11-16T14:14:31,869 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 557ca50786bc4d96a44dab2505671b38, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731766467246 2024-11-16T14:14:31,870 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dea3772ca694b678f123c04a5534c8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731766469091 2024-11-16T14:14:31,870 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f618164f43964365b0c51e3c759d7c23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731766469198 2024-11-16T14:14:31,871 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f0ab4c837804cbb88f8aaebabb8129a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731766469881 2024-11-16T14:14:31,872 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/A of fd241767f9d916c6c1f6b781eb2b8ff7 into 115e123d65344f8584951dca94dccd4f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:31,872 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:31,872 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/A, priority=12, startTime=1731766471800; duration=0sec 2024-11-16T14:14:31,872 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:31,872 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:31,882 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#C#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:31,883 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/a48accb4c1a94e049eddb856f7a26adb is 50, key is test_row_0/C:col10/1731766470526/Put/seqid=0 2024-11-16T14:14:31,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741922_1098 (size=13221) 2024-11-16T14:14:31,917 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/a48accb4c1a94e049eddb856f7a26adb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a48accb4c1a94e049eddb856f7a26adb 2024-11-16T14:14:31,931 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/C of fd241767f9d916c6c1f6b781eb2b8ff7 into a48accb4c1a94e049eddb856f7a26adb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:31,931 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:31,931 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/C, priority=12, startTime=1731766471800; duration=0sec 2024-11-16T14:14:31,931 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:31,931 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:32,258 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/dacb5f9cf8ed499186a0e29c1dabf097 2024-11-16T14:14:32,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/c8d39ec1dc454e2a887014ca7e403813 is 50, key is test_row_0/C:col10/1731766470570/Put/seqid=0 2024-11-16T14:14:32,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741923_1099 (size=12301) 2024-11-16T14:14:32,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T14:14:32,676 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/c8d39ec1dc454e2a887014ca7e403813 2024-11-16T14:14:32,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/50d0c530a6e542d790721433b70c5ba5 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/50d0c530a6e542d790721433b70c5ba5 2024-11-16T14:14:32,689 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/50d0c530a6e542d790721433b70c5ba5, entries=150, sequenceid=517, filesize=12.0 K 2024-11-16T14:14:32,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/dacb5f9cf8ed499186a0e29c1dabf097 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/dacb5f9cf8ed499186a0e29c1dabf097 2024-11-16T14:14:32,701 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/dacb5f9cf8ed499186a0e29c1dabf097, entries=150, sequenceid=517, filesize=12.0 K 2024-11-16T14:14:32,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/c8d39ec1dc454e2a887014ca7e403813 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c8d39ec1dc454e2a887014ca7e403813 2024-11-16T14:14:32,710 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c8d39ec1dc454e2a887014ca7e403813, entries=150, sequenceid=517, filesize=12.0 K 2024-11-16T14:14:32,711 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for fd241767f9d916c6c1f6b781eb2b8ff7 in 898ms, sequenceid=517, compaction requested=false 2024-11-16T14:14:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-16T14:14:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-16T14:14:32,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-16T14:14:32,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5140 sec 2024-11-16T14:14:32,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5190 sec 2024-11-16T14:14:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:32,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:14:32,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:32,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:32,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:32,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:32,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:32,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:32,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/cc1d762262a74641a918df07bef50832 is 50, key is test_row_0/A:col10/1731766472751/Put/seqid=0 2024-11-16T14:14:32,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741924_1100 (size=17181) 2024-11-16T14:14:32,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/cc1d762262a74641a918df07bef50832 2024-11-16T14:14:32,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/f6b7220fb77e46b2babd6bbac8ff81f7 is 50, key is test_row_0/B:col10/1731766472751/Put/seqid=0 2024-11-16T14:14:32,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741925_1101 (size=12301) 2024-11-16T14:14:32,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766472801, max timestamp=1731766472840Num uniques 859; 2024-11-16T14:14:32,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:14:32,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766472801, max timestamp=1731766472840Num uniques 859; 2024-11-16T14:14:32,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:14:32,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766472801, max timestamp=1731766472840Num uniques 859; 2024-11-16T14:14:32,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:14:32,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766532921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:33,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766533035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:33,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/f6b7220fb77e46b2babd6bbac8ff81f7 2024-11-16T14:14:33,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/217000b4b2d941918cb81ee7177b891a is 50, key is test_row_0/C:col10/1731766472751/Put/seqid=0 2024-11-16T14:14:33,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741926_1102 (size=12301) 2024-11-16T14:14:33,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:33,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766533256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:33,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T14:14:33,346 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:33,346 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-16T14:14:33,348 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T14:14:33,350 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:33,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:33,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T14:14:33,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T14:14:33,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:33,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:33,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:33,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:33,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:33,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766533576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:33,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57780 deadline: 1731766533586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:33,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57828 deadline: 1731766533586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:33,589 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in fd241767f9d916c6c1f6b781eb2b8ff7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:33,590 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in fd241767f9d916c6c1f6b781eb2b8ff7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:33,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T14:14:33,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:33,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:33,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:33,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:33,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:33,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/217000b4b2d941918cb81ee7177b891a 2024-11-16T14:14:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T14:14:33,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/cc1d762262a74641a918df07bef50832 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/cc1d762262a74641a918df07bef50832 2024-11-16T14:14:33,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/cc1d762262a74641a918df07bef50832, entries=250, sequenceid=530, filesize=16.8 K 2024-11-16T14:14:33,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/f6b7220fb77e46b2babd6bbac8ff81f7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/f6b7220fb77e46b2babd6bbac8ff81f7 2024-11-16T14:14:33,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/f6b7220fb77e46b2babd6bbac8ff81f7, entries=150, sequenceid=530, filesize=12.0 K 2024-11-16T14:14:33,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/217000b4b2d941918cb81ee7177b891a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/217000b4b2d941918cb81ee7177b891a 2024-11-16T14:14:33,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/217000b4b2d941918cb81ee7177b891a, entries=150, sequenceid=530, filesize=12.0 K 2024-11-16T14:14:33,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fd241767f9d916c6c1f6b781eb2b8ff7 in 928ms, sequenceid=530, compaction requested=true 2024-11-16T14:14:33,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:33,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:33,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:33,688 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:33,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:33,688 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:33,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:33,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:33,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:33,689 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:33,689 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:33,689 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/B is initiating minor compaction (all files) 2024-11-16T14:14:33,689 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/A is initiating minor compaction (all files) 2024-11-16T14:14:33,689 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/A in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:33,689 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/B in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:33,690 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/c4b468e7bef54e84959bcc6eaa07b951, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/dacb5f9cf8ed499186a0e29c1dabf097, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/f6b7220fb77e46b2babd6bbac8ff81f7] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=36.9 K 2024-11-16T14:14:33,690 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/115e123d65344f8584951dca94dccd4f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/50d0c530a6e542d790721433b70c5ba5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/cc1d762262a74641a918df07bef50832] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=41.7 K 2024-11-16T14:14:33,690 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting c4b468e7bef54e84959bcc6eaa07b951, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731766469881 2024-11-16T14:14:33,690 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 115e123d65344f8584951dca94dccd4f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731766469881 2024-11-16T14:14:33,690 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting dacb5f9cf8ed499186a0e29c1dabf097, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1731766470570 2024-11-16T14:14:33,690 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50d0c530a6e542d790721433b70c5ba5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1731766470570 2024-11-16T14:14:33,691 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f6b7220fb77e46b2babd6bbac8ff81f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1731766472749 2024-11-16T14:14:33,691 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc1d762262a74641a918df07bef50832, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1731766472749 2024-11-16T14:14:33,699 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:33,699 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/df957141156a4f88a749bba69a2ca8e2 is 50, key is test_row_0/B:col10/1731766472751/Put/seqid=0 2024-11-16T14:14:33,702 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:33,703 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a92b9ca37d694a06829feba70eef8abd is 50, key is test_row_0/A:col10/1731766472751/Put/seqid=0 2024-11-16T14:14:33,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741928_1104 (size=13323) 2024-11-16T14:14:33,717 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a92b9ca37d694a06829feba70eef8abd as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a92b9ca37d694a06829feba70eef8abd 2024-11-16T14:14:33,728 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/A of fd241767f9d916c6c1f6b781eb2b8ff7 into a92b9ca37d694a06829feba70eef8abd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:33,728 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:33,728 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/A, priority=13, startTime=1731766473688; duration=0sec 2024-11-16T14:14:33,728 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:33,728 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:33,728 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:33,729 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:33,729 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/C is initiating minor compaction (all files) 2024-11-16T14:14:33,730 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/C in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:33,730 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a48accb4c1a94e049eddb856f7a26adb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c8d39ec1dc454e2a887014ca7e403813, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/217000b4b2d941918cb81ee7177b891a] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=36.9 K 2024-11-16T14:14:33,730 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a48accb4c1a94e049eddb856f7a26adb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731766469881 2024-11-16T14:14:33,730 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8d39ec1dc454e2a887014ca7e403813, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1731766470570 2024-11-16T14:14:33,732 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 217000b4b2d941918cb81ee7177b891a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1731766472749 2024-11-16T14:14:33,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741927_1103 (size=13323) 2024-11-16T14:14:33,742 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:33,742 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/f70388a4d30a4c2ca8c724818f62c944 is 50, key is test_row_0/C:col10/1731766472751/Put/seqid=0 2024-11-16T14:14:33,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741929_1105 (size=13323) 2024-11-16T14:14:33,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T14:14:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:33,816 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T14:14:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:33,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/ed527162a05c43fbbafc4bc0978f4713 is 50, key is test_row_0/A:col10/1731766472917/Put/seqid=0 2024-11-16T14:14:33,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741930_1106 (size=12301) 2024-11-16T14:14:33,826 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/ed527162a05c43fbbafc4bc0978f4713 2024-11-16T14:14:33,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/9dd40a2393fa410d8e811dcc6d4dc4cb is 50, key is test_row_0/B:col10/1731766472917/Put/seqid=0 2024-11-16T14:14:33,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741931_1107 (size=12301) 2024-11-16T14:14:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T14:14:34,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:34,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:34,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:34,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766534118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:34,140 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/df957141156a4f88a749bba69a2ca8e2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/df957141156a4f88a749bba69a2ca8e2 2024-11-16T14:14:34,146 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/B of fd241767f9d916c6c1f6b781eb2b8ff7 into df957141156a4f88a749bba69a2ca8e2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:34,146 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:34,146 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/B, priority=13, startTime=1731766473688; duration=0sec 2024-11-16T14:14:34,146 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:34,146 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:34,152 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/f70388a4d30a4c2ca8c724818f62c944 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/f70388a4d30a4c2ca8c724818f62c944 2024-11-16T14:14:34,158 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/C of fd241767f9d916c6c1f6b781eb2b8ff7 into f70388a4d30a4c2ca8c724818f62c944(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:34,158 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:34,158 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/C, priority=13, startTime=1731766473688; duration=0sec 2024-11-16T14:14:34,158 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:34,158 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:34,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:34,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766534225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:34,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:34,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:34,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:34,241 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/9dd40a2393fa410d8e811dcc6d4dc4cb 2024-11-16T14:14:34,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/a66a2dd1a37543dfbbceb243a2afd98c is 50, key is test_row_0/C:col10/1731766472917/Put/seqid=0 2024-11-16T14:14:34,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741932_1108 (size=12301) 2024-11-16T14:14:34,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766534436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:34,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:34,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T14:14:34,658 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/a66a2dd1a37543dfbbceb243a2afd98c 2024-11-16T14:14:34,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/ed527162a05c43fbbafc4bc0978f4713 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed527162a05c43fbbafc4bc0978f4713 2024-11-16T14:14:34,672 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed527162a05c43fbbafc4bc0978f4713, entries=150, sequenceid=566, filesize=12.0 K 2024-11-16T14:14:34,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/9dd40a2393fa410d8e811dcc6d4dc4cb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/9dd40a2393fa410d8e811dcc6d4dc4cb 2024-11-16T14:14:34,684 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/9dd40a2393fa410d8e811dcc6d4dc4cb, entries=150, sequenceid=566, filesize=12.0 K 2024-11-16T14:14:34,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/a66a2dd1a37543dfbbceb243a2afd98c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a66a2dd1a37543dfbbceb243a2afd98c 2024-11-16T14:14:34,695 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a66a2dd1a37543dfbbceb243a2afd98c, entries=150, sequenceid=566, filesize=12.0 K 2024-11-16T14:14:34,702 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for fd241767f9d916c6c1f6b781eb2b8ff7 in 886ms, sequenceid=566, compaction requested=false 2024-11-16T14:14:34,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:34,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:34,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-16T14:14:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-16T14:14:34,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-16T14:14:34,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3550 sec 2024-11-16T14:14:34,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.3630 sec 2024-11-16T14:14:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:34,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:14:34,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:34,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:34,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:34,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:34,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:34,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:34,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a386c7744950415f8eafc6ff1d2f2dd1 is 50, key is test_row_0/A:col10/1731766474117/Put/seqid=0 2024-11-16T14:14:34,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741933_1109 (size=14741) 2024-11-16T14:14:34,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a386c7744950415f8eafc6ff1d2f2dd1 2024-11-16T14:14:34,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/086574590c17433f94f728d6241e27ea is 50, key is test_row_0/B:col10/1731766474117/Put/seqid=0 2024-11-16T14:14:34,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741934_1110 (size=12301) 2024-11-16T14:14:34,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/086574590c17433f94f728d6241e27ea 2024-11-16T14:14:34,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/8adc7d2c2c404faf8c11764e80b70a5c is 50, key is test_row_0/C:col10/1731766474117/Put/seqid=0 2024-11-16T14:14:34,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766474803, max timestamp=1731766474884Num uniques 859; 2024-11-16T14:14:34,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:14:34,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766474803, max timestamp=1731766474884Num uniques 859; 2024-11-16T14:14:34,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:34,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741935_1111 (size=12301) 2024-11-16T14:14:34,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/8adc7d2c2c404faf8c11764e80b70a5c 2024-11-16T14:14:34,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766474803, max timestamp=1731766474884Num uniques 859; 2024-11-16T14:14:34,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/a386c7744950415f8eafc6ff1d2f2dd1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a386c7744950415f8eafc6ff1d2f2dd1 2024-11-16T14:14:34,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a386c7744950415f8eafc6ff1d2f2dd1, entries=200, sequenceid=579, filesize=14.4 K 2024-11-16T14:14:34,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/086574590c17433f94f728d6241e27ea as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/086574590c17433f94f728d6241e27ea 2024-11-16T14:14:34,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/086574590c17433f94f728d6241e27ea, entries=150, sequenceid=579, filesize=12.0 K 2024-11-16T14:14:34,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/8adc7d2c2c404faf8c11764e80b70a5c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/8adc7d2c2c404faf8c11764e80b70a5c 2024-11-16T14:14:34,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/8adc7d2c2c404faf8c11764e80b70a5c, entries=150, sequenceid=579, filesize=12.0 K 2024-11-16T14:14:34,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=84.98 KB/87020 for fd241767f9d916c6c1f6b781eb2b8ff7 in 232ms, sequenceid=579, compaction requested=true 2024-11-16T14:14:34,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:34,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:34,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:34,986 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:34,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:34,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:34,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:34,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T14:14:34,986 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:34,987 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:34,987 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/A is initiating minor compaction (all files) 2024-11-16T14:14:34,987 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/A in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:34,987 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a92b9ca37d694a06829feba70eef8abd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed527162a05c43fbbafc4bc0978f4713, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a386c7744950415f8eafc6ff1d2f2dd1] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=39.4 K 2024-11-16T14:14:34,988 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:34,988 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/B is initiating minor compaction (all files) 2024-11-16T14:14:34,988 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/B in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:34,988 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/df957141156a4f88a749bba69a2ca8e2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/9dd40a2393fa410d8e811dcc6d4dc4cb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/086574590c17433f94f728d6241e27ea] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=37.0 K 2024-11-16T14:14:34,988 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting a92b9ca37d694a06829feba70eef8abd, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1731766472749 2024-11-16T14:14:34,989 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting df957141156a4f88a749bba69a2ca8e2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1731766472749 2024-11-16T14:14:34,989 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ed527162a05c43fbbafc4bc0978f4713, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=566, earliestPutTs=1731766472904 2024-11-16T14:14:34,989 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dd40a2393fa410d8e811dcc6d4dc4cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=566, earliestPutTs=1731766472904 2024-11-16T14:14:34,989 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting a386c7744950415f8eafc6ff1d2f2dd1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731766474103 2024-11-16T14:14:34,989 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 086574590c17433f94f728d6241e27ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731766474103 2024-11-16T14:14:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:35,004 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:35,005 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/73aa5aee2c9c4eb6b8d9b0801638c146 is 50, key is test_row_0/A:col10/1731766474117/Put/seqid=0 2024-11-16T14:14:35,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=105.11 KB heapSize=295.53 KB 2024-11-16T14:14:35,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:35,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:35,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:35,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:35,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:35,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:35,015 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:35,015 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/7b02e8c6470d4d5dbaf2617f0cfe02e8 is 50, key is test_row_0/B:col10/1731766474117/Put/seqid=0 2024-11-16T14:14:35,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d3f968f263964947958a00941a225edb is 50, key is test_row_0/A:col10/1731766474935/Put/seqid=0 2024-11-16T14:14:35,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741936_1112 (size=13425) 2024-11-16T14:14:35,061 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/73aa5aee2c9c4eb6b8d9b0801638c146 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/73aa5aee2c9c4eb6b8d9b0801638c146 2024-11-16T14:14:35,069 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/A of fd241767f9d916c6c1f6b781eb2b8ff7 into 73aa5aee2c9c4eb6b8d9b0801638c146(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:35,069 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:35,069 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/A, priority=13, startTime=1731766474985; duration=0sec 2024-11-16T14:14:35,069 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:35,070 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:35,070 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:35,071 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:35,071 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/C is initiating minor compaction (all files) 2024-11-16T14:14:35,072 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/C in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:35,072 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/f70388a4d30a4c2ca8c724818f62c944, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a66a2dd1a37543dfbbceb243a2afd98c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/8adc7d2c2c404faf8c11764e80b70a5c] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=37.0 K 2024-11-16T14:14:35,073 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f70388a4d30a4c2ca8c724818f62c944, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1731766472749 2024-11-16T14:14:35,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741937_1113 (size=13425) 2024-11-16T14:14:35,073 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting a66a2dd1a37543dfbbceb243a2afd98c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=566, earliestPutTs=1731766472904 2024-11-16T14:14:35,074 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8adc7d2c2c404faf8c11764e80b70a5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731766474103 2024-11-16T14:14:35,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741938_1114 (size=14741) 2024-11-16T14:14:35,087 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/7b02e8c6470d4d5dbaf2617f0cfe02e8 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7b02e8c6470d4d5dbaf2617f0cfe02e8 2024-11-16T14:14:35,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766535089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:35,096 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/B of fd241767f9d916c6c1f6b781eb2b8ff7 into 7b02e8c6470d4d5dbaf2617f0cfe02e8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:35,096 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:35,096 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/B, priority=13, startTime=1731766474986; duration=0sec 2024-11-16T14:14:35,097 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:35,097 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:35,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d3f968f263964947958a00941a225edb 2024-11-16T14:14:35,099 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#C#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:35,099 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/fd2b2f1a016d4b50828187923efa08c0 is 50, key is test_row_0/C:col10/1731766474117/Put/seqid=0 2024-11-16T14:14:35,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/7b66d7213270408dbc491813dd7977a2 is 50, key is test_row_0/B:col10/1731766474935/Put/seqid=0 2024-11-16T14:14:35,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741939_1115 (size=13425) 2024-11-16T14:14:35,119 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/fd2b2f1a016d4b50828187923efa08c0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/fd2b2f1a016d4b50828187923efa08c0 2024-11-16T14:14:35,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741940_1116 (size=12301) 2024-11-16T14:14:35,130 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/C of fd241767f9d916c6c1f6b781eb2b8ff7 into fd2b2f1a016d4b50828187923efa08c0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:35,130 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:35,131 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/C, priority=13, startTime=1731766474986; duration=0sec 2024-11-16T14:14:35,131 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:35,131 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:35,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766535195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:35,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766535406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:35,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57788 deadline: 1731766535455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:35,456 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in fd241767f9d916c6c1f6b781eb2b8ff7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:35,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57772 deadline: 1731766535466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:35,468 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in fd241767f9d916c6c1f6b781eb2b8ff7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T14:14:35,496 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:35,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-16T14:14:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T14:14:35,498 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:35,499 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:35,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:35,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/7b66d7213270408dbc491813dd7977a2 2024-11-16T14:14:35,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/1bcdeba54c414cd9b5587d2a9e43ca8b is 50, key is test_row_0/C:col10/1731766474935/Put/seqid=0 2024-11-16T14:14:35,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741941_1117 (size=12301) 2024-11-16T14:14:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T14:14:35,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T14:14:35,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:35,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:35,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:35,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:35,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:35,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1731766535716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:14:35,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T14:14:35,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:35,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:35,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:35,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:35,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T14:14:35,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/1bcdeba54c414cd9b5587d2a9e43ca8b 2024-11-16T14:14:35,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d3f968f263964947958a00941a225edb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d3f968f263964947958a00941a225edb 2024-11-16T14:14:35,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d3f968f263964947958a00941a225edb, entries=200, sequenceid=611, filesize=14.4 K 2024-11-16T14:14:35,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/7b66d7213270408dbc491813dd7977a2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7b66d7213270408dbc491813dd7977a2 2024-11-16T14:14:35,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7b66d7213270408dbc491813dd7977a2, entries=150, sequenceid=611, filesize=12.0 K 2024-11-16T14:14:35,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/1bcdeba54c414cd9b5587d2a9e43ca8b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1bcdeba54c414cd9b5587d2a9e43ca8b 2024-11-16T14:14:35,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T14:14:35,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:35,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:35,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:35,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:35,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:35,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1bcdeba54c414cd9b5587d2a9e43ca8b, entries=150, sequenceid=611, filesize=12.0 K 2024-11-16T14:14:35,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~105.11 KB/107630, heapSize ~295.48 KB/302576, currentSize=73.80 KB/75570 for fd241767f9d916c6c1f6b781eb2b8ff7 in 956ms, sequenceid=611, compaction requested=false 2024-11-16T14:14:35,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:36,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T14:14:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:36,113 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T14:14:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:36,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d1367f3e5452458bbf2b18c8e6f752ae is 50, key is test_row_0/A:col10/1731766475043/Put/seqid=0 2024-11-16T14:14:36,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741942_1118 (size=12301) 2024-11-16T14:14:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T14:14:36,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. as already flushing 2024-11-16T14:14:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:36,249 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-16T14:14:36,249 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:14:36,249 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-16T14:14:36,249 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:14:36,252 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-16T14:14:36,252 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:14:36,253 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:36,253 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:36,253 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:36,254 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:36,254 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:36,254 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:14:36,254 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:14:36,254 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-16T14:14:36,254 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:14:36,254 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:36,255 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:36,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:14:36,255 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:36,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:14:36,257 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-16T14:14:36,257 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:14:36,257 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:36,257 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:36,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:14:36,526 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d1367f3e5452458bbf2b18c8e6f752ae 2024-11-16T14:14:36,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/193cb28e8e17471c9b07c471340bca4e is 50, key is test_row_0/B:col10/1731766475043/Put/seqid=0 2024-11-16T14:14:36,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741943_1119 (size=12301) 2024-11-16T14:14:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T14:14:36,720 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:14:36,944 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/193cb28e8e17471c9b07c471340bca4e 2024-11-16T14:14:36,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/3c81b6c57cac45d6b050196aa6ed4770 is 50, key is test_row_0/C:col10/1731766475043/Put/seqid=0 2024-11-16T14:14:36,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741944_1120 (size=12301) 2024-11-16T14:14:37,361 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/3c81b6c57cac45d6b050196aa6ed4770 2024-11-16T14:14:37,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d1367f3e5452458bbf2b18c8e6f752ae as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d1367f3e5452458bbf2b18c8e6f752ae 2024-11-16T14:14:37,371 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d1367f3e5452458bbf2b18c8e6f752ae, entries=150, sequenceid=628, filesize=12.0 K 2024-11-16T14:14:37,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/193cb28e8e17471c9b07c471340bca4e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/193cb28e8e17471c9b07c471340bca4e 2024-11-16T14:14:37,377 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/193cb28e8e17471c9b07c471340bca4e, entries=150, sequenceid=628, filesize=12.0 K 2024-11-16T14:14:37,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/3c81b6c57cac45d6b050196aa6ed4770 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/3c81b6c57cac45d6b050196aa6ed4770 2024-11-16T14:14:37,383 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/3c81b6c57cac45d6b050196aa6ed4770, entries=150, sequenceid=628, filesize=12.0 K 2024-11-16T14:14:37,384 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=46.96 KB/48090 for fd241767f9d916c6c1f6b781eb2b8ff7 in 1270ms, sequenceid=628, compaction requested=true 2024-11-16T14:14:37,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:37,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:37,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-16T14:14:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-16T14:14:37,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-16T14:14:37,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8860 sec 2024-11-16T14:14:37,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.8900 sec 2024-11-16T14:14:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T14:14:37,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:43,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:14:43,637 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-16T14:14:43,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:43,637 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:14:43,638 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:43,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:43,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:43,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:43,638 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:43,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:14:43,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:43,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:43,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/598cf82cccf34186b20e8099e0c0d9ce is 50, key is test_row_0/A:col10/1731766476247/Put/seqid=0 2024-11-16T14:14:43,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741945_1121 (size=12301) 2024-11-16T14:14:43,677 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-16T14:14:43,677 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:14:43,677 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:43,677 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:43,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:14:44,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/598cf82cccf34186b20e8099e0c0d9ce 2024-11-16T14:14:44,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/505c2abfb27f401291b9d98fd614c9f3 is 50, key is test_row_0/B:col10/1731766476247/Put/seqid=0 2024-11-16T14:14:44,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741946_1122 (size=12301) 2024-11-16T14:14:44,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/505c2abfb27f401291b9d98fd614c9f3 2024-11-16T14:14:44,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/c31a5e47965d4584ae06f04ca798b439 is 50, key is test_row_0/C:col10/1731766476247/Put/seqid=0 2024-11-16T14:14:44,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741947_1123 (size=12301) 2024-11-16T14:14:44,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/c31a5e47965d4584ae06f04ca798b439 2024-11-16T14:14:44,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/598cf82cccf34186b20e8099e0c0d9ce as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/598cf82cccf34186b20e8099e0c0d9ce 2024-11-16T14:14:44,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/598cf82cccf34186b20e8099e0c0d9ce, entries=150, sequenceid=639, filesize=12.0 K 2024-11-16T14:14:44,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/505c2abfb27f401291b9d98fd614c9f3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/505c2abfb27f401291b9d98fd614c9f3 2024-11-16T14:14:44,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/505c2abfb27f401291b9d98fd614c9f3, entries=150, sequenceid=639, filesize=12.0 K 2024-11-16T14:14:44,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/c31a5e47965d4584ae06f04ca798b439 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c31a5e47965d4584ae06f04ca798b439 2024-11-16T14:14:44,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c31a5e47965d4584ae06f04ca798b439, entries=150, sequenceid=639, filesize=12.0 K 2024-11-16T14:14:44,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=6.71 KB/6870 for fd241767f9d916c6c1f6b781eb2b8ff7 in 1285ms, sequenceid=639, compaction requested=true 2024-11-16T14:14:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:44,922 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:14:44,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:44,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:44,923 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:14:44,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fd241767f9d916c6c1f6b781eb2b8ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:44,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:44,924 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:14:44,924 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:14:44,924 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/A is initiating minor compaction (all files) 2024-11-16T14:14:44,924 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/B is initiating minor compaction (all files) 2024-11-16T14:14:44,924 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/A in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:44,924 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/B in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:44,924 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/73aa5aee2c9c4eb6b8d9b0801638c146, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d3f968f263964947958a00941a225edb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d1367f3e5452458bbf2b18c8e6f752ae, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/598cf82cccf34186b20e8099e0c0d9ce] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=51.5 K 2024-11-16T14:14:44,924 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7b02e8c6470d4d5dbaf2617f0cfe02e8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7b66d7213270408dbc491813dd7977a2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/193cb28e8e17471c9b07c471340bca4e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/505c2abfb27f401291b9d98fd614c9f3] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=49.1 K 2024-11-16T14:14:44,925 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73aa5aee2c9c4eb6b8d9b0801638c146, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731766474103 2024-11-16T14:14:44,925 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b02e8c6470d4d5dbaf2617f0cfe02e8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731766474103 2024-11-16T14:14:44,925 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3f968f263964947958a00941a225edb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1731766474935 2024-11-16T14:14:44,925 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b66d7213270408dbc491813dd7977a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1731766474935 2024-11-16T14:14:44,925 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1367f3e5452458bbf2b18c8e6f752ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1731766475043 2024-11-16T14:14:44,925 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 193cb28e8e17471c9b07c471340bca4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1731766475043 2024-11-16T14:14:44,926 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 598cf82cccf34186b20e8099e0c0d9ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1731766476237 2024-11-16T14:14:44,926 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 505c2abfb27f401291b9d98fd614c9f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1731766476237 2024-11-16T14:14:44,936 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:44,936 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#A#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:44,936 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/2f93a8d7482a4ab1bb8ae89343ed38a1 is 50, key is test_row_0/B:col10/1731766476247/Put/seqid=0 2024-11-16T14:14:44,936 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d9dc8a59740e4cd880dd65c3844fd936 is 50, key is test_row_0/A:col10/1731766476247/Put/seqid=0 2024-11-16T14:14:44,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741949_1125 (size=13561) 2024-11-16T14:14:44,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741948_1124 (size=13561) 2024-11-16T14:14:45,349 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/d9dc8a59740e4cd880dd65c3844fd936 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d9dc8a59740e4cd880dd65c3844fd936 2024-11-16T14:14:45,349 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/2f93a8d7482a4ab1bb8ae89343ed38a1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/2f93a8d7482a4ab1bb8ae89343ed38a1 2024-11-16T14:14:45,355 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/B of fd241767f9d916c6c1f6b781eb2b8ff7 into 2f93a8d7482a4ab1bb8ae89343ed38a1(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:45,355 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/A of fd241767f9d916c6c1f6b781eb2b8ff7 into d9dc8a59740e4cd880dd65c3844fd936(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:45,355 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:45,355 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:45,355 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/A, priority=12, startTime=1731766484922; duration=0sec 2024-11-16T14:14:45,355 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/B, priority=12, startTime=1731766484922; duration=0sec 2024-11-16T14:14:45,356 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:45,356 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:A 2024-11-16T14:14:45,356 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:45,356 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:14:45,356 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:B 2024-11-16T14:14:45,357 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:14:45,357 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fd241767f9d916c6c1f6b781eb2b8ff7/C is initiating minor compaction (all files) 2024-11-16T14:14:45,357 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fd241767f9d916c6c1f6b781eb2b8ff7/C in TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:45,357 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/fd2b2f1a016d4b50828187923efa08c0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1bcdeba54c414cd9b5587d2a9e43ca8b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/3c81b6c57cac45d6b050196aa6ed4770, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c31a5e47965d4584ae06f04ca798b439] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp, totalSize=49.1 K 2024-11-16T14:14:45,358 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd2b2f1a016d4b50828187923efa08c0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731766474103 2024-11-16T14:14:45,358 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bcdeba54c414cd9b5587d2a9e43ca8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1731766474935 2024-11-16T14:14:45,358 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c81b6c57cac45d6b050196aa6ed4770, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1731766475043 2024-11-16T14:14:45,359 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c31a5e47965d4584ae06f04ca798b439, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1731766476237 2024-11-16T14:14:45,368 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fd241767f9d916c6c1f6b781eb2b8ff7#C#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:45,368 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/89324a7507b04916a979dca1ea9c7014 is 50, key is test_row_0/C:col10/1731766476247/Put/seqid=0 2024-11-16T14:14:45,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741950_1126 (size=13561) 2024-11-16T14:14:45,508 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-16T14:14:45,508 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:14:45,508 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:45,508 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:45,508 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:14:45,569 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-16T14:14:45,569 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:14:45,569 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:45,569 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:45,569 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:14:45,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T14:14:45,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-16T14:14:45,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-16T14:14:45,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 280 2024-11-16T14:14:45,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-16T14:14:45,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-16T14:14:45,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T14:14:45,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2707 2024-11-16T14:14:45,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2611 2024-11-16T14:14:45,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T14:14:45,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1175 2024-11-16T14:14:45,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3521 rows 2024-11-16T14:14:45,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1179 2024-11-16T14:14:45,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3536 rows 2024-11-16T14:14:45,571 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:14:45,571 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:14:45,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:45,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:45,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:14:45,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T14:14:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T14:14:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T14:14:45,586 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766485585"}]},"ts":"1731766485585"} 2024-11-16T14:14:45,588 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T14:14:45,588 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T14:14:45,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:14:45,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fd241767f9d916c6c1f6b781eb2b8ff7, UNASSIGN}] 2024-11-16T14:14:45,595 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fd241767f9d916c6c1f6b781eb2b8ff7, UNASSIGN 2024-11-16T14:14:45,596 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=fd241767f9d916c6c1f6b781eb2b8ff7, regionState=CLOSING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:45,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fd241767f9d916c6c1f6b781eb2b8ff7, UNASSIGN because future has completed 2024-11-16T14:14:45,599 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:14:45,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:14:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T14:14:45,756 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:45,756 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:14:45,757 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing fd241767f9d916c6c1f6b781eb2b8ff7, disabling compactions & flushes 2024-11-16T14:14:45,757 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:45,781 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/89324a7507b04916a979dca1ea9c7014 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/89324a7507b04916a979dca1ea9c7014 2024-11-16T14:14:45,790 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fd241767f9d916c6c1f6b781eb2b8ff7/C of fd241767f9d916c6c1f6b781eb2b8ff7 into 89324a7507b04916a979dca1ea9c7014(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:45,790 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fd241767f9d916c6c1f6b781eb2b8ff7: 2024-11-16T14:14:45,790 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:45,791 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., storeName=fd241767f9d916c6c1f6b781eb2b8ff7/C, priority=12, startTime=1731766484923; duration=0sec 2024-11-16T14:14:45,791 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:45,791 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:45,791 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fd241767f9d916c6c1f6b781eb2b8ff7:C 2024-11-16T14:14:45,791 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. after waiting 0 ms 2024-11-16T14:14:45,791 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:45,791 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing fd241767f9d916c6c1f6b781eb2b8ff7 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-16T14:14:45,791 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=A 2024-11-16T14:14:45,791 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:45,792 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=B 2024-11-16T14:14:45,792 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:45,792 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fd241767f9d916c6c1f6b781eb2b8ff7, store=C 2024-11-16T14:14:45,792 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:45,797 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/e0bb7da1b16c4dd0bb892f0504a016e2 is 50, key is test_row_0/A:col10/1731766483676/Put/seqid=0 2024-11-16T14:14:45,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741951_1127 (size=9857) 2024-11-16T14:14:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T14:14:46,204 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=648 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/e0bb7da1b16c4dd0bb892f0504a016e2 2024-11-16T14:14:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T14:14:46,221 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/eae1afc241644bea95d3ca2c2db41f37 is 50, key is test_row_0/B:col10/1731766483676/Put/seqid=0 2024-11-16T14:14:46,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741952_1128 (size=9857) 2024-11-16T14:14:46,629 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=648 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/eae1afc241644bea95d3ca2c2db41f37 2024-11-16T14:14:46,642 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/71f69bfe3ca447bd92230dff2533a79b is 50, key is test_row_0/C:col10/1731766483676/Put/seqid=0 2024-11-16T14:14:46,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741953_1129 (size=9857) 2024-11-16T14:14:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T14:14:47,048 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=648 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/71f69bfe3ca447bd92230dff2533a79b 2024-11-16T14:14:47,056 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/A/e0bb7da1b16c4dd0bb892f0504a016e2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/e0bb7da1b16c4dd0bb892f0504a016e2 2024-11-16T14:14:47,061 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/e0bb7da1b16c4dd0bb892f0504a016e2, entries=100, sequenceid=648, filesize=9.6 K 2024-11-16T14:14:47,062 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/B/eae1afc241644bea95d3ca2c2db41f37 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/eae1afc241644bea95d3ca2c2db41f37 2024-11-16T14:14:47,066 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/eae1afc241644bea95d3ca2c2db41f37, entries=100, sequenceid=648, filesize=9.6 K 2024-11-16T14:14:47,067 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/.tmp/C/71f69bfe3ca447bd92230dff2533a79b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/71f69bfe3ca447bd92230dff2533a79b 2024-11-16T14:14:47,072 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/71f69bfe3ca447bd92230dff2533a79b, entries=100, sequenceid=648, filesize=9.6 K 2024-11-16T14:14:47,073 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fd241767f9d916c6c1f6b781eb2b8ff7 in 1282ms, sequenceid=648, compaction requested=false 2024-11-16T14:14:47,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/1035b488445e42c49800699e880594e2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/18f0c92bc5c04aeebcc2b16f3ed7bbac, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/4dd357149d7c47998ac7f464b9a91899, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/6e7a49004a6c42a7a7dac9cba330122d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed579eab3d9149dbaaddbf612317985d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/f5839350f67a440b894e7a3385166ab9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/366f065c0c514b9d8ccea2b384e6f799, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/0a9e4019bcdd481aabb1e9f81ebf4979, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/00fcd91098a04fc6962211c5847b5d2f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/45295560950446d0952b9503d1ff5934, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a1078b14a68647b4a2dcdb1bb1b42ce5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d19f226072c3473f91fb5d1578c0c7b1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/5859f1fed38648b49546dc991abafbc0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/f677586147234d9aab4efc39db77d557, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed1f896bcff64267b858138d96de8dda, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/da5b4bc49bab4c27aa1850d40d2cbfee, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ccac29a3b055450c8c6dd7e144f7e129, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/535c28db97a645179ab03ad1880f9153, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a26bb1f903bf41d794b5ef21972d3c84, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/e783eed8adda437aa5ad9504cc752098, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/82dc7bb6673c4a45832afd60461a5dbb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/6e4c597aafa9495b8cdecc9b3a31da5b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/cf75286ff9a247c09126468277e2b8df, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a453b70530894d33b91e1d54a5558fe3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/1b0b61bce0644baf8bc22fc72f867f14, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d32d5d7b58234543885b2e439f0fdb55, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/0611ef8e0354499c96a1499d947dadc7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/115e123d65344f8584951dca94dccd4f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/50d0c530a6e542d790721433b70c5ba5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/cc1d762262a74641a918df07bef50832, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a92b9ca37d694a06829feba70eef8abd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed527162a05c43fbbafc4bc0978f4713, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a386c7744950415f8eafc6ff1d2f2dd1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/73aa5aee2c9c4eb6b8d9b0801638c146, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d3f968f263964947958a00941a225edb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d1367f3e5452458bbf2b18c8e6f752ae, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/598cf82cccf34186b20e8099e0c0d9ce] to archive 2024-11-16T14:14:47,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:14:47,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/1035b488445e42c49800699e880594e2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/1035b488445e42c49800699e880594e2 2024-11-16T14:14:47,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/18f0c92bc5c04aeebcc2b16f3ed7bbac to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/18f0c92bc5c04aeebcc2b16f3ed7bbac 2024-11-16T14:14:47,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/4dd357149d7c47998ac7f464b9a91899 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/4dd357149d7c47998ac7f464b9a91899 2024-11-16T14:14:47,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/6e7a49004a6c42a7a7dac9cba330122d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/6e7a49004a6c42a7a7dac9cba330122d 2024-11-16T14:14:47,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed579eab3d9149dbaaddbf612317985d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed579eab3d9149dbaaddbf612317985d 2024-11-16T14:14:47,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/f5839350f67a440b894e7a3385166ab9 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/f5839350f67a440b894e7a3385166ab9 2024-11-16T14:14:47,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/366f065c0c514b9d8ccea2b384e6f799 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/366f065c0c514b9d8ccea2b384e6f799 2024-11-16T14:14:47,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/0a9e4019bcdd481aabb1e9f81ebf4979 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/0a9e4019bcdd481aabb1e9f81ebf4979 2024-11-16T14:14:47,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/00fcd91098a04fc6962211c5847b5d2f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/00fcd91098a04fc6962211c5847b5d2f 2024-11-16T14:14:47,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/45295560950446d0952b9503d1ff5934 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/45295560950446d0952b9503d1ff5934 2024-11-16T14:14:47,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a1078b14a68647b4a2dcdb1bb1b42ce5 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a1078b14a68647b4a2dcdb1bb1b42ce5 2024-11-16T14:14:47,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d19f226072c3473f91fb5d1578c0c7b1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d19f226072c3473f91fb5d1578c0c7b1 2024-11-16T14:14:47,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/5859f1fed38648b49546dc991abafbc0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/5859f1fed38648b49546dc991abafbc0 2024-11-16T14:14:47,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/f677586147234d9aab4efc39db77d557 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/f677586147234d9aab4efc39db77d557 2024-11-16T14:14:47,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed1f896bcff64267b858138d96de8dda to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed1f896bcff64267b858138d96de8dda 2024-11-16T14:14:47,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/da5b4bc49bab4c27aa1850d40d2cbfee to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/da5b4bc49bab4c27aa1850d40d2cbfee 2024-11-16T14:14:47,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ccac29a3b055450c8c6dd7e144f7e129 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ccac29a3b055450c8c6dd7e144f7e129 2024-11-16T14:14:47,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/535c28db97a645179ab03ad1880f9153 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/535c28db97a645179ab03ad1880f9153 2024-11-16T14:14:47,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a26bb1f903bf41d794b5ef21972d3c84 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a26bb1f903bf41d794b5ef21972d3c84 2024-11-16T14:14:47,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/e783eed8adda437aa5ad9504cc752098 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/e783eed8adda437aa5ad9504cc752098 2024-11-16T14:14:47,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/82dc7bb6673c4a45832afd60461a5dbb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/82dc7bb6673c4a45832afd60461a5dbb 2024-11-16T14:14:47,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/6e4c597aafa9495b8cdecc9b3a31da5b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/6e4c597aafa9495b8cdecc9b3a31da5b 2024-11-16T14:14:47,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/cf75286ff9a247c09126468277e2b8df to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/cf75286ff9a247c09126468277e2b8df 2024-11-16T14:14:47,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a453b70530894d33b91e1d54a5558fe3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a453b70530894d33b91e1d54a5558fe3 2024-11-16T14:14:47,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/1b0b61bce0644baf8bc22fc72f867f14 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/1b0b61bce0644baf8bc22fc72f867f14 2024-11-16T14:14:47,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d32d5d7b58234543885b2e439f0fdb55 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d32d5d7b58234543885b2e439f0fdb55 2024-11-16T14:14:47,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/0611ef8e0354499c96a1499d947dadc7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/0611ef8e0354499c96a1499d947dadc7 2024-11-16T14:14:47,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/115e123d65344f8584951dca94dccd4f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/115e123d65344f8584951dca94dccd4f 2024-11-16T14:14:47,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/50d0c530a6e542d790721433b70c5ba5 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/50d0c530a6e542d790721433b70c5ba5 2024-11-16T14:14:47,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/cc1d762262a74641a918df07bef50832 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/cc1d762262a74641a918df07bef50832 2024-11-16T14:14:47,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a92b9ca37d694a06829feba70eef8abd to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a92b9ca37d694a06829feba70eef8abd 2024-11-16T14:14:47,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed527162a05c43fbbafc4bc0978f4713 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/ed527162a05c43fbbafc4bc0978f4713 2024-11-16T14:14:47,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a386c7744950415f8eafc6ff1d2f2dd1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/a386c7744950415f8eafc6ff1d2f2dd1 2024-11-16T14:14:47,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/73aa5aee2c9c4eb6b8d9b0801638c146 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/73aa5aee2c9c4eb6b8d9b0801638c146 2024-11-16T14:14:47,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d3f968f263964947958a00941a225edb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d3f968f263964947958a00941a225edb 2024-11-16T14:14:47,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d1367f3e5452458bbf2b18c8e6f752ae to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d1367f3e5452458bbf2b18c8e6f752ae 2024-11-16T14:14:47,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/598cf82cccf34186b20e8099e0c0d9ce to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/598cf82cccf34186b20e8099e0c0d9ce 2024-11-16T14:14:47,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/3383694bc41244079f5efd3e292ae90e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/335df2fa2c1d4c868640163c9a905aa2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7f1394528b544e90abf0d65c242aba45, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/1878281c4422420182487c8a15de1290, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/a0b7109a7fd348b287ee976eb5c57922, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/ef54d13a0eeb4ccbbab23b753644bd47, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/8795b11a4c64465cb888fbbb10fa293f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/a41f459a4cbe423c8dfa5fc7ec926762, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/2a822ebda6ee442cb5474a8d2b39cf37, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/b95d4680cda14247abb33a299a6aa76f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/d0a10e9593a44de89f773ea81b454a81, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/636b4771bc0a45fca658b5062f4362b3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/5de909dd7dc84b0aba56d683c019af1a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/cd3a8e154a624bf1ae1bbd29c9975d3d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/de77e3d9d3e14962863433f6135fc1ba, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/70b19cfee85a487abf6484634664cf28, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/60fbbd2a86c4445289f12b65c1c679cc, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/1456f97e291b45848242d6bed42aa167, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/cac012d7fdd64c8e8c9c349878b230d4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/919e697ee3774b35b1154995244150f7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/ecf0192daf0f44e1aeb6faacba88adf8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/76d44a752305481ea9efd56f98638cf7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/db5ea9e4f6474cc79ebe6fd5be81ea37, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/3af381cb626e455ba82ddb3dfc9454a9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/e470a4a36eb84d0dafa67cd7e520e731, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/8b2a2b7bd1c746e387ff78c92c86d6d7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/c4b468e7bef54e84959bcc6eaa07b951, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/76317b3e7b904743b164ae4859d59c25, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/dacb5f9cf8ed499186a0e29c1dabf097, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/df957141156a4f88a749bba69a2ca8e2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/f6b7220fb77e46b2babd6bbac8ff81f7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/9dd40a2393fa410d8e811dcc6d4dc4cb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7b02e8c6470d4d5dbaf2617f0cfe02e8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/086574590c17433f94f728d6241e27ea, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7b66d7213270408dbc491813dd7977a2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/193cb28e8e17471c9b07c471340bca4e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/505c2abfb27f401291b9d98fd614c9f3] to archive 2024-11-16T14:14:47,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:14:47,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/3383694bc41244079f5efd3e292ae90e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/3383694bc41244079f5efd3e292ae90e 2024-11-16T14:14:47,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/335df2fa2c1d4c868640163c9a905aa2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/335df2fa2c1d4c868640163c9a905aa2 2024-11-16T14:14:47,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7f1394528b544e90abf0d65c242aba45 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7f1394528b544e90abf0d65c242aba45 2024-11-16T14:14:47,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/1878281c4422420182487c8a15de1290 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/1878281c4422420182487c8a15de1290 2024-11-16T14:14:47,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/a0b7109a7fd348b287ee976eb5c57922 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/a0b7109a7fd348b287ee976eb5c57922 2024-11-16T14:14:47,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/ef54d13a0eeb4ccbbab23b753644bd47 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/ef54d13a0eeb4ccbbab23b753644bd47 2024-11-16T14:14:47,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/8795b11a4c64465cb888fbbb10fa293f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/8795b11a4c64465cb888fbbb10fa293f 2024-11-16T14:14:47,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/a41f459a4cbe423c8dfa5fc7ec926762 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/a41f459a4cbe423c8dfa5fc7ec926762 2024-11-16T14:14:47,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/2a822ebda6ee442cb5474a8d2b39cf37 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/2a822ebda6ee442cb5474a8d2b39cf37 2024-11-16T14:14:47,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/b95d4680cda14247abb33a299a6aa76f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/b95d4680cda14247abb33a299a6aa76f 2024-11-16T14:14:47,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/d0a10e9593a44de89f773ea81b454a81 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/d0a10e9593a44de89f773ea81b454a81 2024-11-16T14:14:47,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/636b4771bc0a45fca658b5062f4362b3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/636b4771bc0a45fca658b5062f4362b3 2024-11-16T14:14:47,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/5de909dd7dc84b0aba56d683c019af1a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/5de909dd7dc84b0aba56d683c019af1a 2024-11-16T14:14:47,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/cd3a8e154a624bf1ae1bbd29c9975d3d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/cd3a8e154a624bf1ae1bbd29c9975d3d 2024-11-16T14:14:47,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/de77e3d9d3e14962863433f6135fc1ba to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/de77e3d9d3e14962863433f6135fc1ba 2024-11-16T14:14:47,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/70b19cfee85a487abf6484634664cf28 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/70b19cfee85a487abf6484634664cf28 2024-11-16T14:14:47,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/60fbbd2a86c4445289f12b65c1c679cc to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/60fbbd2a86c4445289f12b65c1c679cc 2024-11-16T14:14:47,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/1456f97e291b45848242d6bed42aa167 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/1456f97e291b45848242d6bed42aa167 2024-11-16T14:14:47,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/cac012d7fdd64c8e8c9c349878b230d4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/cac012d7fdd64c8e8c9c349878b230d4 2024-11-16T14:14:47,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/919e697ee3774b35b1154995244150f7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/919e697ee3774b35b1154995244150f7 2024-11-16T14:14:47,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/ecf0192daf0f44e1aeb6faacba88adf8 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/ecf0192daf0f44e1aeb6faacba88adf8 2024-11-16T14:14:47,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/76d44a752305481ea9efd56f98638cf7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/76d44a752305481ea9efd56f98638cf7 2024-11-16T14:14:47,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/db5ea9e4f6474cc79ebe6fd5be81ea37 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/db5ea9e4f6474cc79ebe6fd5be81ea37 2024-11-16T14:14:47,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/3af381cb626e455ba82ddb3dfc9454a9 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/3af381cb626e455ba82ddb3dfc9454a9 2024-11-16T14:14:47,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/e470a4a36eb84d0dafa67cd7e520e731 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/e470a4a36eb84d0dafa67cd7e520e731 2024-11-16T14:14:47,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/8b2a2b7bd1c746e387ff78c92c86d6d7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/8b2a2b7bd1c746e387ff78c92c86d6d7 2024-11-16T14:14:47,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/c4b468e7bef54e84959bcc6eaa07b951 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/c4b468e7bef54e84959bcc6eaa07b951 2024-11-16T14:14:47,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/76317b3e7b904743b164ae4859d59c25 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/76317b3e7b904743b164ae4859d59c25 2024-11-16T14:14:47,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/dacb5f9cf8ed499186a0e29c1dabf097 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/dacb5f9cf8ed499186a0e29c1dabf097 2024-11-16T14:14:47,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/df957141156a4f88a749bba69a2ca8e2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/df957141156a4f88a749bba69a2ca8e2 2024-11-16T14:14:47,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/f6b7220fb77e46b2babd6bbac8ff81f7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/f6b7220fb77e46b2babd6bbac8ff81f7 2024-11-16T14:14:47,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/9dd40a2393fa410d8e811dcc6d4dc4cb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/9dd40a2393fa410d8e811dcc6d4dc4cb 2024-11-16T14:14:47,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7b02e8c6470d4d5dbaf2617f0cfe02e8 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7b02e8c6470d4d5dbaf2617f0cfe02e8 2024-11-16T14:14:47,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/086574590c17433f94f728d6241e27ea to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/086574590c17433f94f728d6241e27ea 2024-11-16T14:14:47,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7b66d7213270408dbc491813dd7977a2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/7b66d7213270408dbc491813dd7977a2 2024-11-16T14:14:47,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/193cb28e8e17471c9b07c471340bca4e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/193cb28e8e17471c9b07c471340bca4e 2024-11-16T14:14:47,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/505c2abfb27f401291b9d98fd614c9f3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/505c2abfb27f401291b9d98fd614c9f3 2024-11-16T14:14:47,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/504adb91d46b48fc8c6c690adc8ebe61, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/49ba03208fb34c0486f00357272efc55, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/ff42ac1eb41c491288852ada8abffbf4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/0ac3cab406e54a508d25feecfa1660e5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1d760dab85f64acea5d74919927b72e0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a8d7510ef01f41298504184a00922dee, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/e11853d53b56499284ff5ef8b1bc5fbb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/0867ac5532d8421390d17946103b1baa, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/30294b91dfa74d81b0c5a1fd0c172e1d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/904c1a6777ec44e5aa657d2e8858c7c2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d9653ad20fd743f0865f502be606730a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7059ff4d02d24aea91c9c1327ead885d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1ec2ca37de4a41338627b3bec571a22b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7c07f934f1794bdcb82a349d9765760f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/176c105b4de549e59bbd517fecc26b2e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/93e94397d63a4021a2bb3b2d3dca5ab8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/610d8579c3414206bbaab4dde7da4a88, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d455820eeb1e4c51afc492e3cf259b85, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c65ef5dd0ec645aa80f365eb8fc99cd0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7d4de58d41a84a93958deb62e1fbcbcd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d793b8551f994c5fb19c474be3b556a4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/71c379258d244e94909434591d66a04f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/557ca50786bc4d96a44dab2505671b38, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/aa341c0f35c3412ea558005c2a6e283b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7dea3772ca694b678f123c04a5534c8e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/f618164f43964365b0c51e3c759d7c23, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a48accb4c1a94e049eddb856f7a26adb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/9f0ab4c837804cbb88f8aaebabb8129a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c8d39ec1dc454e2a887014ca7e403813, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/f70388a4d30a4c2ca8c724818f62c944, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/217000b4b2d941918cb81ee7177b891a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a66a2dd1a37543dfbbceb243a2afd98c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/fd2b2f1a016d4b50828187923efa08c0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/8adc7d2c2c404faf8c11764e80b70a5c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1bcdeba54c414cd9b5587d2a9e43ca8b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/3c81b6c57cac45d6b050196aa6ed4770, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c31a5e47965d4584ae06f04ca798b439] to archive 2024-11-16T14:14:47,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:14:47,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/504adb91d46b48fc8c6c690adc8ebe61 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/504adb91d46b48fc8c6c690adc8ebe61 2024-11-16T14:14:47,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/49ba03208fb34c0486f00357272efc55 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/49ba03208fb34c0486f00357272efc55 2024-11-16T14:14:47,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/ff42ac1eb41c491288852ada8abffbf4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/ff42ac1eb41c491288852ada8abffbf4 2024-11-16T14:14:47,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/0ac3cab406e54a508d25feecfa1660e5 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/0ac3cab406e54a508d25feecfa1660e5 2024-11-16T14:14:47,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1d760dab85f64acea5d74919927b72e0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1d760dab85f64acea5d74919927b72e0 2024-11-16T14:14:47,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a8d7510ef01f41298504184a00922dee to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a8d7510ef01f41298504184a00922dee 2024-11-16T14:14:47,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/e11853d53b56499284ff5ef8b1bc5fbb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/e11853d53b56499284ff5ef8b1bc5fbb 2024-11-16T14:14:47,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/0867ac5532d8421390d17946103b1baa to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/0867ac5532d8421390d17946103b1baa 2024-11-16T14:14:47,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/30294b91dfa74d81b0c5a1fd0c172e1d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/30294b91dfa74d81b0c5a1fd0c172e1d 2024-11-16T14:14:47,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/904c1a6777ec44e5aa657d2e8858c7c2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/904c1a6777ec44e5aa657d2e8858c7c2 2024-11-16T14:14:47,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d9653ad20fd743f0865f502be606730a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d9653ad20fd743f0865f502be606730a 2024-11-16T14:14:47,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7059ff4d02d24aea91c9c1327ead885d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7059ff4d02d24aea91c9c1327ead885d 2024-11-16T14:14:47,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1ec2ca37de4a41338627b3bec571a22b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1ec2ca37de4a41338627b3bec571a22b 2024-11-16T14:14:47,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7c07f934f1794bdcb82a349d9765760f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7c07f934f1794bdcb82a349d9765760f 2024-11-16T14:14:47,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/176c105b4de549e59bbd517fecc26b2e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/176c105b4de549e59bbd517fecc26b2e 2024-11-16T14:14:47,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/93e94397d63a4021a2bb3b2d3dca5ab8 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/93e94397d63a4021a2bb3b2d3dca5ab8 2024-11-16T14:14:47,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/610d8579c3414206bbaab4dde7da4a88 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/610d8579c3414206bbaab4dde7da4a88 2024-11-16T14:14:47,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d455820eeb1e4c51afc492e3cf259b85 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d455820eeb1e4c51afc492e3cf259b85 2024-11-16T14:14:47,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c65ef5dd0ec645aa80f365eb8fc99cd0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c65ef5dd0ec645aa80f365eb8fc99cd0 2024-11-16T14:14:47,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7d4de58d41a84a93958deb62e1fbcbcd to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7d4de58d41a84a93958deb62e1fbcbcd 2024-11-16T14:14:47,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d793b8551f994c5fb19c474be3b556a4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/d793b8551f994c5fb19c474be3b556a4 2024-11-16T14:14:47,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/71c379258d244e94909434591d66a04f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/71c379258d244e94909434591d66a04f 2024-11-16T14:14:47,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/557ca50786bc4d96a44dab2505671b38 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/557ca50786bc4d96a44dab2505671b38 2024-11-16T14:14:47,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/aa341c0f35c3412ea558005c2a6e283b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/aa341c0f35c3412ea558005c2a6e283b 2024-11-16T14:14:47,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7dea3772ca694b678f123c04a5534c8e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/7dea3772ca694b678f123c04a5534c8e 2024-11-16T14:14:47,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/f618164f43964365b0c51e3c759d7c23 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/f618164f43964365b0c51e3c759d7c23 2024-11-16T14:14:47,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a48accb4c1a94e049eddb856f7a26adb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a48accb4c1a94e049eddb856f7a26adb 2024-11-16T14:14:47,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/9f0ab4c837804cbb88f8aaebabb8129a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/9f0ab4c837804cbb88f8aaebabb8129a 2024-11-16T14:14:47,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c8d39ec1dc454e2a887014ca7e403813 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c8d39ec1dc454e2a887014ca7e403813 2024-11-16T14:14:47,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/f70388a4d30a4c2ca8c724818f62c944 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/f70388a4d30a4c2ca8c724818f62c944 2024-11-16T14:14:47,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/217000b4b2d941918cb81ee7177b891a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/217000b4b2d941918cb81ee7177b891a 2024-11-16T14:14:47,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a66a2dd1a37543dfbbceb243a2afd98c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/a66a2dd1a37543dfbbceb243a2afd98c 2024-11-16T14:14:47,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/fd2b2f1a016d4b50828187923efa08c0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/fd2b2f1a016d4b50828187923efa08c0 2024-11-16T14:14:47,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/8adc7d2c2c404faf8c11764e80b70a5c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/8adc7d2c2c404faf8c11764e80b70a5c 2024-11-16T14:14:47,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1bcdeba54c414cd9b5587d2a9e43ca8b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/1bcdeba54c414cd9b5587d2a9e43ca8b 2024-11-16T14:14:47,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/3c81b6c57cac45d6b050196aa6ed4770 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/3c81b6c57cac45d6b050196aa6ed4770 2024-11-16T14:14:47,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c31a5e47965d4584ae06f04ca798b439 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/c31a5e47965d4584ae06f04ca798b439 2024-11-16T14:14:47,227 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/recovered.edits/651.seqid, newMaxSeqId=651, maxSeqId=1 2024-11-16T14:14:47,228 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. 2024-11-16T14:14:47,228 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for fd241767f9d916c6c1f6b781eb2b8ff7: Waiting for close lock at 1731766485757Running coprocessor pre-close hooks at 1731766485757Disabling compacts and flushes for region at 1731766485757Disabling writes for close at 1731766485791 (+34 ms)Obtaining lock to block concurrent updates at 1731766485791Preparing flush snapshotting stores in fd241767f9d916c6c1f6b781eb2b8ff7 at 1731766485791Finished memstore snapshotting TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731766485792 (+1 ms)Flushing stores of TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7. at 1731766485793 (+1 ms)Flushing fd241767f9d916c6c1f6b781eb2b8ff7/A: creating writer at 1731766485793Flushing fd241767f9d916c6c1f6b781eb2b8ff7/A: appending metadata at 1731766485797 (+4 ms)Flushing fd241767f9d916c6c1f6b781eb2b8ff7/A: closing flushed file at 1731766485797Flushing fd241767f9d916c6c1f6b781eb2b8ff7/B: creating writer at 1731766486216 (+419 ms)Flushing fd241767f9d916c6c1f6b781eb2b8ff7/B: appending metadata at 1731766486220 (+4 ms)Flushing fd241767f9d916c6c1f6b781eb2b8ff7/B: closing flushed file at 1731766486220Flushing fd241767f9d916c6c1f6b781eb2b8ff7/C: creating writer at 1731766486639 (+419 ms)Flushing fd241767f9d916c6c1f6b781eb2b8ff7/C: appending metadata at 1731766486641 (+2 ms)Flushing fd241767f9d916c6c1f6b781eb2b8ff7/C: closing flushed file at 1731766486641Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c1ec9d0: reopening flushed file at 1731766487055 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@323cc1fe: reopening flushed file at 1731766487061 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47d57a5d: reopening flushed file at 1731766487067 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fd241767f9d916c6c1f6b781eb2b8ff7 in 1282ms, sequenceid=648, compaction requested=false at 1731766487073 (+6 ms)Writing region close event to WAL at 1731766487224 (+151 ms)Running coprocessor post-close hooks at 1731766487228 (+4 ms)Closed at 1731766487228 2024-11-16T14:14:47,230 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:47,231 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=fd241767f9d916c6c1f6b781eb2b8ff7, regionState=CLOSED 2024-11-16T14:14:47,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:14:47,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-16T14:14:47,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure fd241767f9d916c6c1f6b781eb2b8ff7, server=04e93f2b5fad,41317,1731766448342 in 1.6340 sec 2024-11-16T14:14:47,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-16T14:14:47,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fd241767f9d916c6c1f6b781eb2b8ff7, UNASSIGN in 1.6410 sec 2024-11-16T14:14:47,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-16T14:14:47,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6480 sec 2024-11-16T14:14:47,240 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766487240"}]},"ts":"1731766487240"} 2024-11-16T14:14:47,242 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T14:14:47,242 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T14:14:47,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6640 sec 2024-11-16T14:14:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T14:14:47,736 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:47,740 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T14:14:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:14:47,748 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:14:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T14:14:47,749 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:14:47,755 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:47,759 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/recovered.edits] 2024-11-16T14:14:47,763 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d9dc8a59740e4cd880dd65c3844fd936 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/d9dc8a59740e4cd880dd65c3844fd936 2024-11-16T14:14:47,765 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/e0bb7da1b16c4dd0bb892f0504a016e2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/A/e0bb7da1b16c4dd0bb892f0504a016e2 2024-11-16T14:14:47,768 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/2f93a8d7482a4ab1bb8ae89343ed38a1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/2f93a8d7482a4ab1bb8ae89343ed38a1 2024-11-16T14:14:47,770 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/eae1afc241644bea95d3ca2c2db41f37 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/B/eae1afc241644bea95d3ca2c2db41f37 2024-11-16T14:14:47,773 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/71f69bfe3ca447bd92230dff2533a79b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/71f69bfe3ca447bd92230dff2533a79b 2024-11-16T14:14:47,774 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/89324a7507b04916a979dca1ea9c7014 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/C/89324a7507b04916a979dca1ea9c7014 2024-11-16T14:14:47,777 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/recovered.edits/651.seqid to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7/recovered.edits/651.seqid 2024-11-16T14:14:47,777 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fd241767f9d916c6c1f6b781eb2b8ff7 2024-11-16T14:14:47,777 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T14:14:47,782 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:14:47,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-16T14:14:47,792 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T14:14:47,827 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T14:14:47,829 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:14:47,829 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T14:14:47,829 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731766487829"}]},"ts":"9223372036854775807"} 2024-11-16T14:14:47,834 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T14:14:47,834 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fd241767f9d916c6c1f6b781eb2b8ff7, NAME => 'TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T14:14:47,835 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T14:14:47,835 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731766487835"}]},"ts":"9223372036854775807"} 2024-11-16T14:14:47,838 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T14:14:47,839 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:14:47,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 98 msec 2024-11-16T14:14:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T14:14:47,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T14:14:47,857 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:47,868 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=250 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1474060206_22 at /127.0.0.1:51702 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-571885164_22 at /127.0.0.1:51706 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS:0;04e93f2b5fad:41317-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-571885164_22 at /127.0.0.1:51674 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1474060206_22 at /127.0.0.1:51708 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=300 (was 237) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4193 (was 4822) 2024-11-16T14:14:47,873 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T14:14:47,877 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=300, ProcessCount=11, AvailableMemoryMB=4191 2024-11-16T14:14:47,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:14:47,879 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:14:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T14:14:47,884 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:14:47,885 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:47,885 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-16T14:14:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T14:14:47,886 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:14:47,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741954_1130 (size=1061) 2024-11-16T14:14:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T14:14:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T14:14:48,298 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc 2024-11-16T14:14:48,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741955_1131 (size=53) 2024-11-16T14:14:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T14:14:48,709 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:14:48,709 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c646687c72e5cbf577a48e3dcdc6f92b, disabling compactions & flushes 2024-11-16T14:14:48,709 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:48,709 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:48,709 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. after waiting 0 ms 2024-11-16T14:14:48,709 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:48,709 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:48,709 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c646687c72e5cbf577a48e3dcdc6f92b: Waiting for close lock at 1731766488709Disabling compacts and flushes for region at 1731766488709Disabling writes for close at 1731766488709Writing region close event to WAL at 1731766488709Closed at 1731766488709 2024-11-16T14:14:48,710 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:14:48,710 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731766488710"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731766488710"}]},"ts":"1731766488710"} 2024-11-16T14:14:48,712 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:14:48,713 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:14:48,713 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766488713"}]},"ts":"1731766488713"} 2024-11-16T14:14:48,714 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T14:14:48,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, ASSIGN}] 2024-11-16T14:14:48,716 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, ASSIGN 2024-11-16T14:14:48,716 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, ASSIGN; state=OFFLINE, location=04e93f2b5fad,41317,1731766448342; forceNewPlan=false, retain=false 2024-11-16T14:14:48,868 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=c646687c72e5cbf577a48e3dcdc6f92b, regionState=OPENING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:48,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, ASSIGN because future has completed 2024-11-16T14:14:48,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:14:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T14:14:49,031 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:49,031 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:14:49,032 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,032 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:14:49,032 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,032 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,034 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,035 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:14:49,036 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c646687c72e5cbf577a48e3dcdc6f92b columnFamilyName A 2024-11-16T14:14:49,036 DEBUG [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:49,037 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.HStore(327): Store=c646687c72e5cbf577a48e3dcdc6f92b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:14:49,037 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,038 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:14:49,039 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c646687c72e5cbf577a48e3dcdc6f92b columnFamilyName B 2024-11-16T14:14:49,039 DEBUG [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:49,039 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.HStore(327): Store=c646687c72e5cbf577a48e3dcdc6f92b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:14:49,040 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,041 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:14:49,041 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c646687c72e5cbf577a48e3dcdc6f92b columnFamilyName C 2024-11-16T14:14:49,041 DEBUG [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:49,042 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.HStore(327): Store=c646687c72e5cbf577a48e3dcdc6f92b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:14:49,042 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:49,042 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,043 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,043 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,045 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,045 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,045 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:14:49,047 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,049 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:14:49,049 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened c646687c72e5cbf577a48e3dcdc6f92b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73905441, jitterRate=0.10127688944339752}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:14:49,049 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:49,050 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for c646687c72e5cbf577a48e3dcdc6f92b: Running coprocessor pre-open hook at 1731766489032Writing region info on filesystem at 1731766489032Initializing all the Stores at 1731766489033 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766489033Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766489033Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766489033Cleaning up temporary data from old regions at 1731766489045 (+12 ms)Running coprocessor post-open hooks at 1731766489049 (+4 ms)Region opened successfully at 1731766489050 (+1 ms) 2024-11-16T14:14:49,051 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., pid=32, masterSystemTime=1731766489027 2024-11-16T14:14:49,053 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:49,054 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:49,054 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=c646687c72e5cbf577a48e3dcdc6f92b, regionState=OPEN, openSeqNum=2, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:49,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:14:49,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-16T14:14:49,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 in 184 msec 2024-11-16T14:14:49,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-16T14:14:49,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, ASSIGN in 345 msec 2024-11-16T14:14:49,063 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:14:49,063 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766489063"}]},"ts":"1731766489063"} 2024-11-16T14:14:49,065 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T14:14:49,066 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:14:49,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-11-16T14:14:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T14:14:50,047 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:50,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736030eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:50,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:50,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:50,054 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:50,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5655b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:50,060 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf8b3a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:50,062 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:50,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:50,067 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:14:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 04e93f2b5fad,39231,1731766447537 2024-11-16T14:14:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ad97762 2024-11-16T14:14:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:14:50,069 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:14:50,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:14:50,075 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:14:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T14:14:50,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741956_1132 (size=1097) 2024-11-16T14:14:50,499 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T14:14:50,499 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T14:14:50,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:14:50,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, REOPEN/MOVE}] 2024-11-16T14:14:50,517 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, REOPEN/MOVE 2024-11-16T14:14:50,518 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c646687c72e5cbf577a48e3dcdc6f92b, regionState=CLOSING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:50,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, REOPEN/MOVE because future has completed 2024-11-16T14:14:50,520 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:14:50,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:14:50,675 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:50,675 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:14:50,675 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing c646687c72e5cbf577a48e3dcdc6f92b, disabling compactions & flushes 2024-11-16T14:14:50,675 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:50,675 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:50,676 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. after waiting 0 ms 2024-11-16T14:14:50,676 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:50,687 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T14:14:50,688 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:50,688 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for c646687c72e5cbf577a48e3dcdc6f92b: Waiting for close lock at 1731766490675Running coprocessor pre-close hooks at 1731766490675Disabling compacts and flushes for region at 1731766490675Disabling writes for close at 1731766490676 (+1 ms)Writing region close event to WAL at 1731766490677 (+1 ms)Running coprocessor post-close hooks at 1731766490687 (+10 ms)Closed at 1731766490687 2024-11-16T14:14:50,688 WARN [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: c646687c72e5cbf577a48e3dcdc6f92b to self. 2024-11-16T14:14:50,690 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:50,691 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c646687c72e5cbf577a48e3dcdc6f92b, regionState=CLOSED 2024-11-16T14:14:50,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:14:50,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-16T14:14:50,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 in 175 msec 2024-11-16T14:14:50,698 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, REOPEN/MOVE; state=CLOSED, location=04e93f2b5fad,41317,1731766448342; forceNewPlan=false, retain=true 2024-11-16T14:14:50,848 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c646687c72e5cbf577a48e3dcdc6f92b, regionState=OPENING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:50,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, REOPEN/MOVE because future has completed 2024-11-16T14:14:50,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:14:51,006 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,006 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:14:51,006 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,006 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:14:51,006 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,006 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,009 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,010 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:14:51,014 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c646687c72e5cbf577a48e3dcdc6f92b columnFamilyName A 2024-11-16T14:14:51,017 DEBUG [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:51,017 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.HStore(327): Store=c646687c72e5cbf577a48e3dcdc6f92b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:14:51,018 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,018 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:14:51,019 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c646687c72e5cbf577a48e3dcdc6f92b columnFamilyName B 2024-11-16T14:14:51,019 DEBUG [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:51,019 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.HStore(327): Store=c646687c72e5cbf577a48e3dcdc6f92b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:14:51,019 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,020 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:14:51,020 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c646687c72e5cbf577a48e3dcdc6f92b columnFamilyName C 2024-11-16T14:14:51,020 DEBUG [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:51,021 INFO [StoreOpener-c646687c72e5cbf577a48e3dcdc6f92b-1 {}] regionserver.HStore(327): Store=c646687c72e5cbf577a48e3dcdc6f92b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:14:51,021 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,021 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,021 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,022 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,023 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,024 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,024 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:14:51,025 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,026 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened c646687c72e5cbf577a48e3dcdc6f92b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59148327, jitterRate=-0.11862124502658844}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:14:51,026 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,027 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for c646687c72e5cbf577a48e3dcdc6f92b: Running coprocessor pre-open hook at 1731766491007Writing region info on filesystem at 1731766491007Initializing all the Stores at 1731766491007Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766491007Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766491008 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766491008Cleaning up temporary data from old regions at 1731766491024 (+16 ms)Running coprocessor post-open hooks at 1731766491026 (+2 ms)Region opened successfully at 1731766491027 (+1 ms) 2024-11-16T14:14:51,028 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., pid=37, masterSystemTime=1731766491002 2024-11-16T14:14:51,030 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,030 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,030 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c646687c72e5cbf577a48e3dcdc6f92b, regionState=OPEN, openSeqNum=5, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:14:51,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-16T14:14:51,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 in 183 msec 2024-11-16T14:14:51,036 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-16T14:14:51,036 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, REOPEN/MOVE in 518 msec 2024-11-16T14:14:51,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-16T14:14:51,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-16T14:14:51,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 963 msec 2024-11-16T14:14:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T14:14:51,044 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:51,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:51,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:51,046 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc8eeab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:51,048 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8ba2e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:51,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:51,050 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17322130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:51,051 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b7d864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208b436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:51,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:51,054 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a373624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:51,055 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c7afaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:51,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:51,057 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@420a968b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:51,059 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c1276f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:51,060 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:51,061 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23aab562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:51,062 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:51,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cd1b799, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:51,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:51,065 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5103819a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:51,066 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f12ec3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,067 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:51,067 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:51,068 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@762206d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:51,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:51,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:51,069 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46bfc8e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,070 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:51,070 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:51,071 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ba5a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:51,072 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679bff6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711c9f1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,073 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:14:51,074 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:14:51,074 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:14:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:14:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b593e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:14:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:14:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:14:51,076 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:14:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@408942e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:14:51,077 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:51,077 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:51,078 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:51,078 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:51,078 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:51,078 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:51,079 DEBUG [Thread-666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:51,080 DEBUG [Thread-668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:51,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:51,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-16T14:14:51,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:51,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T14:14:51,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:51,081 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:51,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:51,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:51,082 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:51,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:51,083 DEBUG [Thread-670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:51,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:51,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:51,084 DEBUG [Thread-679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:14:51,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:14:51,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:14:51,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:51,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:14:51,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:51,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:14:51,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:51,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:14:51,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:14:51,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:14:51,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:14:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:14:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:14:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:14:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:14:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:14:51,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:14:51,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:51,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:14:51,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:51,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:14:51,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:51,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116cde74164fc3d4159b75f609a194faa45_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766491097/Put/seqid=0 2024-11-16T14:14:51,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741957_1133 (size=12154) 2024-11-16T14:14:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T14:14:51,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766491187, max timestamp=1731766491191Num uniques 859; 2024-11-16T14:14:51,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766491187, max timestamp=1731766491191Num uniques 859; 2024-11-16T14:14:51,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:51,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:51,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766491187, max timestamp=1731766491191Num uniques 859; 2024-11-16T14:14:51,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:14:51,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766551220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766551222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T14:14:51,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731766551228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58106 deadline: 1731766551231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766551231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766551335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766551336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58106 deadline: 1731766551346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731766551346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766551347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T14:14:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T14:14:51,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T14:14:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,544 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766551546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766551546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,558 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116cde74164fc3d4159b75f609a194faa45_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cde74164fc3d4159b75f609a194faa45_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:51,559 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/82f66dd8d97a421997618d45216d9c95, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:51,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731766551556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58106 deadline: 1731766551556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766551556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/82f66dd8d97a421997618d45216d9c95 is 175, key is test_row_0/A:col10/1731766491097/Put/seqid=0 2024-11-16T14:14:51,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741958_1134 (size=30951) 2024-11-16T14:14:51,582 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/82f66dd8d97a421997618d45216d9c95 2024-11-16T14:14:51,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/35a90528b7a84f439c08940b0a7b9a76 is 50, key is test_row_0/B:col10/1731766491097/Put/seqid=0 2024-11-16T14:14:51,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741959_1135 (size=9657) 2024-11-16T14:14:51,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T14:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T14:14:51,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T14:14:51,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:51,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:51,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:51,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766551855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766551856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731766551876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766551877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58106 deadline: 1731766551877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T14:14:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:52,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:52,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:52,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/35a90528b7a84f439c08940b0a7b9a76 2024-11-16T14:14:52,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/f31b3b73ec1e4df6bda0762a66eb194b is 50, key is test_row_0/C:col10/1731766491097/Put/seqid=0 2024-11-16T14:14:52,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741960_1136 (size=9657) 2024-11-16T14:14:52,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/f31b3b73ec1e4df6bda0762a66eb194b 2024-11-16T14:14:52,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/82f66dd8d97a421997618d45216d9c95 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/82f66dd8d97a421997618d45216d9c95 2024-11-16T14:14:52,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/82f66dd8d97a421997618d45216d9c95, entries=150, sequenceid=15, filesize=30.2 K 2024-11-16T14:14:52,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/35a90528b7a84f439c08940b0a7b9a76 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/35a90528b7a84f439c08940b0a7b9a76 2024-11-16T14:14:52,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/35a90528b7a84f439c08940b0a7b9a76, entries=100, sequenceid=15, filesize=9.4 K 2024-11-16T14:14:52,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/f31b3b73ec1e4df6bda0762a66eb194b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/f31b3b73ec1e4df6bda0762a66eb194b 2024-11-16T14:14:52,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/f31b3b73ec1e4df6bda0762a66eb194b, entries=100, sequenceid=15, filesize=9.4 K 2024-11-16T14:14:52,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c646687c72e5cbf577a48e3dcdc6f92b in 1019ms, sequenceid=15, compaction requested=false 2024-11-16T14:14:52,118 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T14:14:52,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:52,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T14:14:52,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:52,162 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T14:14:52,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:14:52,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:52,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:14:52,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:52,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:14:52,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:14:52,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d86fb0be13be4edeb613d5df4d7fd994_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766491220/Put/seqid=0 2024-11-16T14:14:52,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741961_1137 (size=12154) 2024-11-16T14:14:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T14:14:52,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:52,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731766552386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58106 deadline: 1731766552387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766552387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766552387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766552388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766552496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766552505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766552505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:52,595 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d86fb0be13be4edeb613d5df4d7fd994_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d86fb0be13be4edeb613d5df4d7fd994_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:52,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/c2322f6fc02d4cf29885fefab9eba526, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:52,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/c2322f6fc02d4cf29885fefab9eba526 is 175, key is test_row_0/A:col10/1731766491220/Put/seqid=0 2024-11-16T14:14:52,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741962_1138 (size=30955) 2024-11-16T14:14:52,602 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/c2322f6fc02d4cf29885fefab9eba526 2024-11-16T14:14:52,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/491fa816fd3a48d9be731df8dda97d37 is 50, key is test_row_0/B:col10/1731766491220/Put/seqid=0 2024-11-16T14:14:52,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741963_1139 (size=12001) 2024-11-16T14:14:52,636 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/491fa816fd3a48d9be731df8dda97d37 2024-11-16T14:14:52,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/ad746a6b45a64b0e8b286b2419206864 is 50, key is test_row_0/C:col10/1731766491220/Put/seqid=0 2024-11-16T14:14:52,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741964_1140 (size=12001) 2024-11-16T14:14:52,654 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/ad746a6b45a64b0e8b286b2419206864 2024-11-16T14:14:52,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/c2322f6fc02d4cf29885fefab9eba526 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/c2322f6fc02d4cf29885fefab9eba526 2024-11-16T14:14:52,668 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/c2322f6fc02d4cf29885fefab9eba526, entries=150, sequenceid=51, filesize=30.2 K 2024-11-16T14:14:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/491fa816fd3a48d9be731df8dda97d37 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/491fa816fd3a48d9be731df8dda97d37 2024-11-16T14:14:52,677 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/491fa816fd3a48d9be731df8dda97d37, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T14:14:52,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/ad746a6b45a64b0e8b286b2419206864 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ad746a6b45a64b0e8b286b2419206864 2024-11-16T14:14:52,688 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ad746a6b45a64b0e8b286b2419206864, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T14:14:52,691 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for c646687c72e5cbf577a48e3dcdc6f92b in 528ms, sequenceid=51, compaction requested=false 2024-11-16T14:14:52,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:52,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:52,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-16T14:14:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-16T14:14:52,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-16T14:14:52,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6110 sec 2024-11-16T14:14:52,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.6180 sec 2024-11-16T14:14:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:52,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:14:52,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:14:52,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:52,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:14:52,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:52,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:14:52,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:52,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f35455774e49478487bc17b86934b190_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766492707/Put/seqid=0 2024-11-16T14:14:52,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741965_1141 (size=12154) 2024-11-16T14:14:52,755 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:52,762 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f35455774e49478487bc17b86934b190_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f35455774e49478487bc17b86934b190_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:52,763 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/985b3d9545d84e49b75b9e341d63fc13, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:52,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/985b3d9545d84e49b75b9e341d63fc13 is 175, key is test_row_0/A:col10/1731766492707/Put/seqid=0 2024-11-16T14:14:52,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766552758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766552763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766552764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,772 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:14:52,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741966_1142 (size=30955) 2024-11-16T14:14:52,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766552875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766552875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:52,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766552876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:53,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766553086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:53,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766553086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:53,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766553086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:53,180 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/985b3d9545d84e49b75b9e341d63fc13 2024-11-16T14:14:53,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/28d8122a8ffe4bd88909c8e288b0006b is 50, key is test_row_0/B:col10/1731766492707/Put/seqid=0 2024-11-16T14:14:53,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741967_1143 (size=12001) 2024-11-16T14:14:53,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T14:14:53,236 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:53,237 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:53,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-16T14:14:53,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T14:14:53,238 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:53,239 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:53,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T14:14:53,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T14:14:53,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:53,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:53,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:53,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:53,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766553396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766553396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766553396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731766553396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:53,398 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in c646687c72e5cbf577a48e3dcdc6f92b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:53,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58106 deadline: 1731766553405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:53,409 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in c646687c72e5cbf577a48e3dcdc6f92b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:53,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:53,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T14:14:53,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:53,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:53,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:53,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:53,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T14:14:53,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/28d8122a8ffe4bd88909c8e288b0006b 2024-11-16T14:14:53,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/991b3640dddc4390acf402f86c65d1ae is 50, key is test_row_0/C:col10/1731766492707/Put/seqid=0 2024-11-16T14:14:53,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741968_1144 (size=12001) 2024-11-16T14:14:53,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/991b3640dddc4390acf402f86c65d1ae 2024-11-16T14:14:53,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/985b3d9545d84e49b75b9e341d63fc13 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/985b3d9545d84e49b75b9e341d63fc13 2024-11-16T14:14:53,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/985b3d9545d84e49b75b9e341d63fc13, entries=150, sequenceid=63, filesize=30.2 K 2024-11-16T14:14:53,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/28d8122a8ffe4bd88909c8e288b0006b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/28d8122a8ffe4bd88909c8e288b0006b 2024-11-16T14:14:53,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/28d8122a8ffe4bd88909c8e288b0006b, entries=150, sequenceid=63, filesize=11.7 K 2024-11-16T14:14:53,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T14:14:53,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/991b3640dddc4390acf402f86c65d1ae as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/991b3640dddc4390acf402f86c65d1ae 2024-11-16T14:14:53,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:53,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:53,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:53,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:53,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:53,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/991b3640dddc4390acf402f86c65d1ae, entries=150, sequenceid=63, filesize=11.7 K 2024-11-16T14:14:53,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c646687c72e5cbf577a48e3dcdc6f92b in 1034ms, sequenceid=63, compaction requested=true 2024-11-16T14:14:53,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:53,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:53,743 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:53,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:53,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:53,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:53,743 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:53,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:53,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:53,744 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:53,744 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:53,744 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/A is initiating minor compaction (all files) 2024-11-16T14:14:53,744 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/B is initiating minor compaction (all files) 2024-11-16T14:14:53,744 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/A in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:53,744 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/B in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:53,744 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/82f66dd8d97a421997618d45216d9c95, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/c2322f6fc02d4cf29885fefab9eba526, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/985b3d9545d84e49b75b9e341d63fc13] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=90.7 K 2024-11-16T14:14:53,744 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/35a90528b7a84f439c08940b0a7b9a76, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/491fa816fd3a48d9be731df8dda97d37, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/28d8122a8ffe4bd88909c8e288b0006b] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=32.9 K 2024-11-16T14:14:53,744 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:53,745 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 35a90528b7a84f439c08940b0a7b9a76, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731766491097 2024-11-16T14:14:53,745 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/82f66dd8d97a421997618d45216d9c95, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/c2322f6fc02d4cf29885fefab9eba526, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/985b3d9545d84e49b75b9e341d63fc13] 2024-11-16T14:14:53,745 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 491fa816fd3a48d9be731df8dda97d37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731766491220 2024-11-16T14:14:53,745 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82f66dd8d97a421997618d45216d9c95, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731766491095 2024-11-16T14:14:53,746 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 28d8122a8ffe4bd88909c8e288b0006b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731766492380 2024-11-16T14:14:53,746 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2322f6fc02d4cf29885fefab9eba526, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731766491220 2024-11-16T14:14:53,746 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 985b3d9545d84e49b75b9e341d63fc13, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731766492380 2024-11-16T14:14:53,773 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#B#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:53,774 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/1617d7dc9b014a6d80222dd6f230376b is 50, key is test_row_0/B:col10/1731766492707/Put/seqid=0 2024-11-16T14:14:53,785 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:53,797 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116ee193ba69bb64cb1bb099752355c404c_c646687c72e5cbf577a48e3dcdc6f92b store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:53,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741969_1145 (size=12104) 2024-11-16T14:14:53,826 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/1617d7dc9b014a6d80222dd6f230376b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/1617d7dc9b014a6d80222dd6f230376b 2024-11-16T14:14:53,830 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116ee193ba69bb64cb1bb099752355c404c_c646687c72e5cbf577a48e3dcdc6f92b, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:53,830 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ee193ba69bb64cb1bb099752355c404c_c646687c72e5cbf577a48e3dcdc6f92b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:53,838 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/B of c646687c72e5cbf577a48e3dcdc6f92b into 1617d7dc9b014a6d80222dd6f230376b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:53,838 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:53,838 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/B, priority=13, startTime=1731766493743; duration=0sec 2024-11-16T14:14:53,838 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:53,838 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:B 2024-11-16T14:14:53,838 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:53,841 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:53,841 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/C is initiating minor compaction (all files) 2024-11-16T14:14:53,841 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/C in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:53,841 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/f31b3b73ec1e4df6bda0762a66eb194b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ad746a6b45a64b0e8b286b2419206864, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/991b3640dddc4390acf402f86c65d1ae] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=32.9 K 2024-11-16T14:14:53,842 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f31b3b73ec1e4df6bda0762a66eb194b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731766491097 2024-11-16T14:14:53,842 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ad746a6b45a64b0e8b286b2419206864, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731766491220 2024-11-16T14:14:53,842 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 991b3640dddc4390acf402f86c65d1ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731766492380 2024-11-16T14:14:53,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T14:14:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:53,861 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:14:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:14:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:14:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:14:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:53,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741970_1146 (size=4469) 2024-11-16T14:14:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T14:14:53,866 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#A#compaction#127 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:53,867 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/0c45988af30240198d892b3ff299d004 is 175, key is test_row_0/A:col10/1731766492707/Put/seqid=0 2024-11-16T14:14:53,871 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#C#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:53,872 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/22f26026b5ff4e039c15972cf4a1c006 is 50, key is test_row_0/C:col10/1731766492707/Put/seqid=0 2024-11-16T14:14:53,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ae57150320ad41a9beed393a66963b2e_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766492745/Put/seqid=0 2024-11-16T14:14:53,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:53,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741971_1147 (size=31058) 2024-11-16T14:14:53,935 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/0c45988af30240198d892b3ff299d004 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/0c45988af30240198d892b3ff299d004 2024-11-16T14:14:53,947 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/A of c646687c72e5cbf577a48e3dcdc6f92b into 0c45988af30240198d892b3ff299d004(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:53,947 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:53,947 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/A, priority=13, startTime=1731766493742; duration=0sec 2024-11-16T14:14:53,947 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:53,947 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:A 2024-11-16T14:14:53,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:53,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766553947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766553948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:53,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766553951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:53,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741973_1149 (size=12154) 2024-11-16T14:14:53,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:53,968 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ae57150320ad41a9beed393a66963b2e_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ae57150320ad41a9beed393a66963b2e_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:53,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/8dff098fc27b4dee9e4ff9e47f1e945c, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:53,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/8dff098fc27b4dee9e4ff9e47f1e945c is 175, key is test_row_0/A:col10/1731766492745/Put/seqid=0 2024-11-16T14:14:53,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741972_1148 (size=12104) 2024-11-16T14:14:54,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741974_1150 (size=30955) 2024-11-16T14:14:54,007 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/8dff098fc27b4dee9e4ff9e47f1e945c 2024-11-16T14:14:54,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/fa72a3f26338493fb9d6024c620d1ecf is 50, key is test_row_0/B:col10/1731766492745/Put/seqid=0 2024-11-16T14:14:54,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741975_1151 (size=12001) 2024-11-16T14:14:54,058 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/fa72a3f26338493fb9d6024c620d1ecf 2024-11-16T14:14:54,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766554067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:54,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766554067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:54,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766554067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:54,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/b9a2ebe3b056405397c9a6909e85a714 is 50, key is test_row_0/C:col10/1731766492745/Put/seqid=0 2024-11-16T14:14:54,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741976_1152 (size=12001) 2024-11-16T14:14:54,105 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/b9a2ebe3b056405397c9a6909e85a714 2024-11-16T14:14:54,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/8dff098fc27b4dee9e4ff9e47f1e945c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8dff098fc27b4dee9e4ff9e47f1e945c 2024-11-16T14:14:54,120 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8dff098fc27b4dee9e4ff9e47f1e945c, entries=150, sequenceid=84, filesize=30.2 K 2024-11-16T14:14:54,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/fa72a3f26338493fb9d6024c620d1ecf as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fa72a3f26338493fb9d6024c620d1ecf 2024-11-16T14:14:54,128 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fa72a3f26338493fb9d6024c620d1ecf, entries=150, sequenceid=84, filesize=11.7 K 2024-11-16T14:14:54,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/b9a2ebe3b056405397c9a6909e85a714 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/b9a2ebe3b056405397c9a6909e85a714 2024-11-16T14:14:54,138 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/b9a2ebe3b056405397c9a6909e85a714, entries=150, sequenceid=84, filesize=11.7 K 2024-11-16T14:14:54,141 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c646687c72e5cbf577a48e3dcdc6f92b in 280ms, sequenceid=84, compaction requested=false 2024-11-16T14:14:54,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:54,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:54,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-16T14:14:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-16T14:14:54,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-16T14:14:54,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 904 msec 2024-11-16T14:14:54,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 914 msec 2024-11-16T14:14:54,225 INFO [master/04e93f2b5fad:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-16T14:14:54,225 INFO [master/04e93f2b5fad:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-16T14:14:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:54,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T14:14:54,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:14:54,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:54,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:14:54,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:54,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:14:54,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:54,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111627a024b92fef494faf12514c040d7227_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766493948/Put/seqid=0 2024-11-16T14:14:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T14:14:54,377 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:54,377 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-16T14:14:54,386 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:14:54,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741977_1153 (size=12154) 2024-11-16T14:14:54,389 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:54,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:54,394 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/22f26026b5ff4e039c15972cf4a1c006 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/22f26026b5ff4e039c15972cf4a1c006 2024-11-16T14:14:54,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766554387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:54,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766554390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:54,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766554399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:54,404 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/C of c646687c72e5cbf577a48e3dcdc6f92b into 22f26026b5ff4e039c15972cf4a1c006(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:54,404 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:54,404 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/C, priority=13, startTime=1731766493743; duration=0sec 2024-11-16T14:14:54,404 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:54,404 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:C 2024-11-16T14:14:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:14:54,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766554506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:54,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766554506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:54,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766554506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:54,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:14:54,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:54,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:54,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:54,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:54,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:54,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:14:54,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:54,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:54,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:54,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:54,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:14:54,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766554716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:54,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766554716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:54,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766554725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:54,788 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:54,792 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111627a024b92fef494faf12514c040d7227_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111627a024b92fef494faf12514c040d7227_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:54,794 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/a73b3377de9c4b6aaf6ac2b9809c53a4, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:54,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/a73b3377de9c4b6aaf6ac2b9809c53a4 is 175, key is test_row_0/A:col10/1731766493948/Put/seqid=0 2024-11-16T14:14:54,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741978_1154 (size=30955) 2024-11-16T14:14:54,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:14:54,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:54,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:54,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:54,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:54,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:14:55,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:55,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:14:55,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:55,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766555026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766555026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:55,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766555035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:55,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:14:55,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:55,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,207 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/a73b3377de9c4b6aaf6ac2b9809c53a4 2024-11-16T14:14:55,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/0b46e832037c44cf85c7cc02fe426fd6 is 50, key is test_row_0/B:col10/1731766493948/Put/seqid=0 2024-11-16T14:14:55,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741979_1155 (size=12001) 2024-11-16T14:14:55,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:14:55,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:55,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:55,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58106 deadline: 1731766555425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731766555425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:55,430 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in c646687c72e5cbf577a48e3dcdc6f92b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,430 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in c646687c72e5cbf577a48e3dcdc6f92b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:55,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:14:55,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:55,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:14:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766555535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:55,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766555546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:55,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766555546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:55,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:14:55,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:55,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/0b46e832037c44cf85c7cc02fe426fd6 2024-11-16T14:14:55,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/f0417496a4854b3e8a354bdf29fbad44 is 50, key is test_row_0/C:col10/1731766493948/Put/seqid=0 2024-11-16T14:14:55,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741980_1156 (size=12001) 2024-11-16T14:14:55,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:14:55,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:55,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:14:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:55,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:14:56,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/f0417496a4854b3e8a354bdf29fbad44 2024-11-16T14:14:56,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/a73b3377de9c4b6aaf6ac2b9809c53a4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/a73b3377de9c4b6aaf6ac2b9809c53a4 2024-11-16T14:14:56,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/a73b3377de9c4b6aaf6ac2b9809c53a4, entries=150, sequenceid=100, filesize=30.2 K 2024-11-16T14:14:56,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/0b46e832037c44cf85c7cc02fe426fd6 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/0b46e832037c44cf85c7cc02fe426fd6 2024-11-16T14:14:56,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/0b46e832037c44cf85c7cc02fe426fd6, entries=150, sequenceid=100, filesize=11.7 K 2024-11-16T14:14:56,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/f0417496a4854b3e8a354bdf29fbad44 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/f0417496a4854b3e8a354bdf29fbad44 2024-11-16T14:14:56,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/f0417496a4854b3e8a354bdf29fbad44, entries=150, sequenceid=100, filesize=11.7 K 2024-11-16T14:14:56,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c646687c72e5cbf577a48e3dcdc6f92b in 1731ms, sequenceid=100, compaction requested=true 2024-11-16T14:14:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:56,054 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:56,054 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:56,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:56,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:56,055 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:56,055 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:56,056 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/A is initiating minor compaction (all files) 2024-11-16T14:14:56,056 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/B is initiating minor compaction (all files) 2024-11-16T14:14:56,056 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/A in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:56,056 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/B in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:56,056 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/0c45988af30240198d892b3ff299d004, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8dff098fc27b4dee9e4ff9e47f1e945c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/a73b3377de9c4b6aaf6ac2b9809c53a4] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=90.8 K 2024-11-16T14:14:56,056 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/1617d7dc9b014a6d80222dd6f230376b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fa72a3f26338493fb9d6024c620d1ecf, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/0b46e832037c44cf85c7cc02fe426fd6] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=35.3 K 2024-11-16T14:14:56,056 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:56,056 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/0c45988af30240198d892b3ff299d004, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8dff098fc27b4dee9e4ff9e47f1e945c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/a73b3377de9c4b6aaf6ac2b9809c53a4] 2024-11-16T14:14:56,056 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1617d7dc9b014a6d80222dd6f230376b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731766492380 2024-11-16T14:14:56,056 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c45988af30240198d892b3ff299d004, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731766492380 2024-11-16T14:14:56,057 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting fa72a3f26338493fb9d6024c620d1ecf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731766492745 2024-11-16T14:14:56,057 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b46e832037c44cf85c7cc02fe426fd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731766493948 2024-11-16T14:14:56,057 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dff098fc27b4dee9e4ff9e47f1e945c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731766492745 2024-11-16T14:14:56,058 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a73b3377de9c4b6aaf6ac2b9809c53a4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731766493948 2024-11-16T14:14:56,065 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:56,071 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#B#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:56,071 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/ba6bd23393224b17a921b3b530d7be26 is 50, key is test_row_0/B:col10/1731766493948/Put/seqid=0 2024-11-16T14:14:56,073 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116b1a8d3b73e374a0b89099384e2ce8f8f_c646687c72e5cbf577a48e3dcdc6f92b store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:56,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:14:56,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:56,075 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T14:14:56,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:14:56,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:56,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:14:56,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:56,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:14:56,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:56,078 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116b1a8d3b73e374a0b89099384e2ce8f8f_c646687c72e5cbf577a48e3dcdc6f92b, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:56,078 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b1a8d3b73e374a0b89099384e2ce8f8f_c646687c72e5cbf577a48e3dcdc6f92b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:56,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741981_1157 (size=12207) 2024-11-16T14:14:56,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d81211871d9b47548af5840a704e0660_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766494370/Put/seqid=0 2024-11-16T14:14:56,098 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/ba6bd23393224b17a921b3b530d7be26 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/ba6bd23393224b17a921b3b530d7be26 2024-11-16T14:14:56,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741982_1158 (size=4469) 2024-11-16T14:14:56,105 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/B of c646687c72e5cbf577a48e3dcdc6f92b into ba6bd23393224b17a921b3b530d7be26(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:56,105 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:56,106 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/B, priority=13, startTime=1731766496054; duration=0sec 2024-11-16T14:14:56,106 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:56,106 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:B 2024-11-16T14:14:56,106 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:56,106 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#A#compaction#135 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:56,107 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/b4697348334e4c32976888d9e1f270be is 175, key is test_row_0/A:col10/1731766493948/Put/seqid=0 2024-11-16T14:14:56,110 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:56,110 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/C is initiating minor compaction (all files) 2024-11-16T14:14:56,110 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/C in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:56,111 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/22f26026b5ff4e039c15972cf4a1c006, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/b9a2ebe3b056405397c9a6909e85a714, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/f0417496a4854b3e8a354bdf29fbad44] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=35.3 K 2024-11-16T14:14:56,111 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 22f26026b5ff4e039c15972cf4a1c006, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731766492380 2024-11-16T14:14:56,112 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a2ebe3b056405397c9a6909e85a714, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731766492745 2024-11-16T14:14:56,113 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f0417496a4854b3e8a354bdf29fbad44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731766493948 2024-11-16T14:14:56,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741983_1159 (size=12154) 2024-11-16T14:14:56,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:56,129 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d81211871d9b47548af5840a704e0660_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d81211871d9b47548af5840a704e0660_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:56,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/34a4b6903b3a43769437509e07c83a44, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:56,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/34a4b6903b3a43769437509e07c83a44 is 175, key is test_row_0/A:col10/1731766494370/Put/seqid=0 2024-11-16T14:14:56,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741984_1160 (size=31161) 2024-11-16T14:14:56,137 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:56,138 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/ae73c26b9921414aa46ea45b9812c12a is 50, key is test_row_0/C:col10/1731766493948/Put/seqid=0 2024-11-16T14:14:56,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741985_1161 (size=30955) 2024-11-16T14:14:56,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741986_1162 (size=12207) 2024-11-16T14:14:56,544 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/b4697348334e4c32976888d9e1f270be as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/b4697348334e4c32976888d9e1f270be 2024-11-16T14:14:56,545 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/34a4b6903b3a43769437509e07c83a44 2024-11-16T14:14:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:14:56,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:56,557 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/A of c646687c72e5cbf577a48e3dcdc6f92b into b4697348334e4c32976888d9e1f270be(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:56,557 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:56,557 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/A, priority=13, startTime=1731766496054; duration=0sec 2024-11-16T14:14:56,557 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:56,557 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:A 2024-11-16T14:14:56,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/9bdbdc8bd2dd4aeea056380c9b745769 is 50, key is test_row_0/B:col10/1731766494370/Put/seqid=0 2024-11-16T14:14:56,568 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/ae73c26b9921414aa46ea45b9812c12a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ae73c26b9921414aa46ea45b9812c12a 2024-11-16T14:14:56,577 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/C of c646687c72e5cbf577a48e3dcdc6f92b into ae73c26b9921414aa46ea45b9812c12a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:56,577 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:56,577 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/C, priority=13, startTime=1731766496054; duration=0sec 2024-11-16T14:14:56,577 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:56,577 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:C 2024-11-16T14:14:56,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741987_1163 (size=12001) 2024-11-16T14:14:56,582 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/9bdbdc8bd2dd4aeea056380c9b745769 2024-11-16T14:14:56,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766556580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:56,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766556580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:56,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/fb8242262aa645b3821eedbec0ac4e05 is 50, key is test_row_0/C:col10/1731766494370/Put/seqid=0 2024-11-16T14:14:56,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766556582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:56,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741988_1164 (size=12001) 2024-11-16T14:14:56,609 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/fb8242262aa645b3821eedbec0ac4e05 2024-11-16T14:14:56,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/34a4b6903b3a43769437509e07c83a44 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/34a4b6903b3a43769437509e07c83a44 2024-11-16T14:14:56,623 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/34a4b6903b3a43769437509e07c83a44, entries=150, sequenceid=118, filesize=30.2 K 2024-11-16T14:14:56,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/9bdbdc8bd2dd4aeea056380c9b745769 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/9bdbdc8bd2dd4aeea056380c9b745769 2024-11-16T14:14:56,633 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/9bdbdc8bd2dd4aeea056380c9b745769, entries=150, sequenceid=118, filesize=11.7 K 2024-11-16T14:14:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/fb8242262aa645b3821eedbec0ac4e05 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fb8242262aa645b3821eedbec0ac4e05 2024-11-16T14:14:56,661 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fb8242262aa645b3821eedbec0ac4e05, entries=150, sequenceid=118, filesize=11.7 K 2024-11-16T14:14:56,662 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for c646687c72e5cbf577a48e3dcdc6f92b in 587ms, sequenceid=118, compaction requested=false 2024-11-16T14:14:56,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:56,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:56,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-16T14:14:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-16T14:14:56,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-16T14:14:56,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2750 sec 2024-11-16T14:14:56,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.2940 sec 2024-11-16T14:14:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:56,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T14:14:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:14:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:14:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:14:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:56,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161fb5d54a60c84b809ab902bc2a34a3cf_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766496688/Put/seqid=0 2024-11-16T14:14:56,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766556711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:56,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766556713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:56,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766556719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:56,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741989_1165 (size=14794) 2024-11-16T14:14:56,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766556825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:56,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766556825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:56,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766556836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:57,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766557035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766557035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:57,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766557046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:57,134 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:57,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161fb5d54a60c84b809ab902bc2a34a3cf_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161fb5d54a60c84b809ab902bc2a34a3cf_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:57,140 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/5247f3df9f3d4092a2f63b0e5ae9d5ab, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:57,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/5247f3df9f3d4092a2f63b0e5ae9d5ab is 175, key is test_row_0/A:col10/1731766496688/Put/seqid=0 2024-11-16T14:14:57,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741990_1166 (size=39749) 2024-11-16T14:14:57,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766557345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:57,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:57,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766557345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:57,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:57,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:57,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766557355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:57,548 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/5247f3df9f3d4092a2f63b0e5ae9d5ab 2024-11-16T14:14:57,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/fc512898c72c4a70a604c02b8295fb03 is 50, key is test_row_0/B:col10/1731766496688/Put/seqid=0 2024-11-16T14:14:57,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741991_1167 (size=12151) 2024-11-16T14:14:57,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/fc512898c72c4a70a604c02b8295fb03 2024-11-16T14:14:57,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/959c09de00464c78a2d801cb48c94a21 is 50, key is test_row_0/C:col10/1731766496688/Put/seqid=0 2024-11-16T14:14:57,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741992_1168 (size=12151) 2024-11-16T14:14:57,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766557855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766557855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:57,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766557865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:57,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:57,873 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T14:14:57,873 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T14:14:57,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/959c09de00464c78a2d801cb48c94a21 2024-11-16T14:14:57,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/5247f3df9f3d4092a2f63b0e5ae9d5ab as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/5247f3df9f3d4092a2f63b0e5ae9d5ab 2024-11-16T14:14:57,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/5247f3df9f3d4092a2f63b0e5ae9d5ab, entries=200, sequenceid=138, filesize=38.8 K 2024-11-16T14:14:57,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/fc512898c72c4a70a604c02b8295fb03 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fc512898c72c4a70a604c02b8295fb03 2024-11-16T14:14:58,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fc512898c72c4a70a604c02b8295fb03, entries=150, sequenceid=138, filesize=11.9 K 2024-11-16T14:14:58,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/959c09de00464c78a2d801cb48c94a21 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/959c09de00464c78a2d801cb48c94a21 2024-11-16T14:14:58,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/959c09de00464c78a2d801cb48c94a21, entries=150, sequenceid=138, filesize=11.9 K 2024-11-16T14:14:58,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c646687c72e5cbf577a48e3dcdc6f92b in 1326ms, sequenceid=138, compaction requested=true 2024-11-16T14:14:58,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:58,017 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:58,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:14:58,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:58,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:14:58,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:58,017 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:58,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:14:58,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:58,018 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:58,018 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/A is initiating minor compaction (all files) 2024-11-16T14:14:58,018 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/A in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:58,019 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/b4697348334e4c32976888d9e1f270be, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/34a4b6903b3a43769437509e07c83a44, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/5247f3df9f3d4092a2f63b0e5ae9d5ab] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=99.5 K 2024-11-16T14:14:58,019 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:58,019 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/b4697348334e4c32976888d9e1f270be, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/34a4b6903b3a43769437509e07c83a44, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/5247f3df9f3d4092a2f63b0e5ae9d5ab] 2024-11-16T14:14:58,020 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:58,020 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4697348334e4c32976888d9e1f270be, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731766493948 2024-11-16T14:14:58,020 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/B is initiating minor compaction (all files) 2024-11-16T14:14:58,021 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/B in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:58,021 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/ba6bd23393224b17a921b3b530d7be26, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/9bdbdc8bd2dd4aeea056380c9b745769, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fc512898c72c4a70a604c02b8295fb03] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=35.5 K 2024-11-16T14:14:58,021 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ba6bd23393224b17a921b3b530d7be26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731766493948 2024-11-16T14:14:58,021 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34a4b6903b3a43769437509e07c83a44, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731766494370 2024-11-16T14:14:58,022 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bdbdc8bd2dd4aeea056380c9b745769, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731766494370 2024-11-16T14:14:58,022 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5247f3df9f3d4092a2f63b0e5ae9d5ab, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731766496568 2024-11-16T14:14:58,022 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting fc512898c72c4a70a604c02b8295fb03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731766496568 2024-11-16T14:14:58,030 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:58,032 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#B#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:58,032 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/20c1d784715f45c58cf76376323d5faa is 50, key is test_row_0/B:col10/1731766496688/Put/seqid=0 2024-11-16T14:14:58,035 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411164227d6c973184808a5e998025b86fb75_c646687c72e5cbf577a48e3dcdc6f92b store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:58,038 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411164227d6c973184808a5e998025b86fb75_c646687c72e5cbf577a48e3dcdc6f92b, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:58,039 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164227d6c973184808a5e998025b86fb75_c646687c72e5cbf577a48e3dcdc6f92b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:58,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741993_1169 (size=12459) 2024-11-16T14:14:58,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741994_1170 (size=4469) 2024-11-16T14:14:58,058 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#A#compaction#145 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:58,059 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/83fcf45f1d3246cbaedc1b91cfb895ef is 175, key is test_row_0/A:col10/1731766496688/Put/seqid=0 2024-11-16T14:14:58,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741995_1171 (size=31413) 2024-11-16T14:14:58,079 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/83fcf45f1d3246cbaedc1b91cfb895ef as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/83fcf45f1d3246cbaedc1b91cfb895ef 2024-11-16T14:14:58,085 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/A of c646687c72e5cbf577a48e3dcdc6f92b into 83fcf45f1d3246cbaedc1b91cfb895ef(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:58,085 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:58,085 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/A, priority=13, startTime=1731766498017; duration=0sec 2024-11-16T14:14:58,085 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:14:58,085 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:A 2024-11-16T14:14:58,085 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:14:58,087 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:14:58,087 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/C is initiating minor compaction (all files) 2024-11-16T14:14:58,087 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/C in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:58,087 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ae73c26b9921414aa46ea45b9812c12a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fb8242262aa645b3821eedbec0ac4e05, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/959c09de00464c78a2d801cb48c94a21] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=35.5 K 2024-11-16T14:14:58,087 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae73c26b9921414aa46ea45b9812c12a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731766493948 2024-11-16T14:14:58,088 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb8242262aa645b3821eedbec0ac4e05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731766494370 2024-11-16T14:14:58,088 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 959c09de00464c78a2d801cb48c94a21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731766496568 2024-11-16T14:14:58,098 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#C#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:14:58,098 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/045fe9d286c348fdab10eb72cc289cc0 is 50, key is test_row_0/C:col10/1731766496688/Put/seqid=0 2024-11-16T14:14:58,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741996_1172 (size=12459) 2024-11-16T14:14:58,453 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/20c1d784715f45c58cf76376323d5faa as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/20c1d784715f45c58cf76376323d5faa 2024-11-16T14:14:58,458 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/B of c646687c72e5cbf577a48e3dcdc6f92b into 20c1d784715f45c58cf76376323d5faa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:58,458 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:58,458 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/B, priority=13, startTime=1731766498017; duration=0sec 2024-11-16T14:14:58,458 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:58,458 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:B 2024-11-16T14:14:58,509 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/045fe9d286c348fdab10eb72cc289cc0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/045fe9d286c348fdab10eb72cc289cc0 2024-11-16T14:14:58,517 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/C of c646687c72e5cbf577a48e3dcdc6f92b into 045fe9d286c348fdab10eb72cc289cc0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:14:58,517 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:58,517 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/C, priority=13, startTime=1731766498018; duration=0sec 2024-11-16T14:14:58,517 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:14:58,517 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:C 2024-11-16T14:14:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:14:58,566 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:58,566 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-16T14:14:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T14:14:58,571 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:58,573 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:58,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T14:14:58,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T14:14:58,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:58,726 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T14:14:58,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:14:58,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:58,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:14:58,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:58,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:14:58,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:58,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fa336c492ea1434e9d506d7461dd83c9_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766496706/Put/seqid=0 2024-11-16T14:14:58,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741997_1173 (size=12304) 2024-11-16T14:14:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:58,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:14:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T14:14:58,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766558897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:58,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766558899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:58,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766558899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:58,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:58,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:58,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:59,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766559015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:59,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766559016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:59,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766559016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:59,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:14:59,155 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fa336c492ea1434e9d506d7461dd83c9_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fa336c492ea1434e9d506d7461dd83c9_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:14:59,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/65aaaff06f274e58a78c82daa4c88567, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:14:59,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/65aaaff06f274e58a78c82daa4c88567 is 175, key is test_row_0/A:col10/1731766496706/Put/seqid=0 2024-11-16T14:14:59,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741998_1174 (size=31105) 2024-11-16T14:14:59,162 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/65aaaff06f274e58a78c82daa4c88567 2024-11-16T14:14:59,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/4c3983059850469ebcd7576eea001546 is 50, key is test_row_0/B:col10/1731766496706/Put/seqid=0 2024-11-16T14:14:59,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741999_1175 (size=12151) 2024-11-16T14:14:59,176 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/4c3983059850469ebcd7576eea001546 2024-11-16T14:14:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T14:14:59,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766559226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:59,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766559226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766559226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:59,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/fe469ebaf1c14ad1b11afb39f900e6f1 is 50, key is test_row_0/C:col10/1731766496706/Put/seqid=0 2024-11-16T14:14:59,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742000_1176 (size=12151) 2024-11-16T14:14:59,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:59,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58106 deadline: 1731766559466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731766559466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:59,467 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in c646687c72e5cbf577a48e3dcdc6f92b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:59,467 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in c646687c72e5cbf577a48e3dcdc6f92b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:14:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:59,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766559535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:59,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766559536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:59,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:14:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766559536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:14:59,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:14:59,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:14:59,663 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/fe469ebaf1c14ad1b11afb39f900e6f1 2024-11-16T14:14:59,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/65aaaff06f274e58a78c82daa4c88567 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/65aaaff06f274e58a78c82daa4c88567 2024-11-16T14:14:59,673 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/65aaaff06f274e58a78c82daa4c88567, entries=150, sequenceid=156, filesize=30.4 K 2024-11-16T14:14:59,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/4c3983059850469ebcd7576eea001546 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/4c3983059850469ebcd7576eea001546 2024-11-16T14:14:59,680 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/4c3983059850469ebcd7576eea001546, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T14:14:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/fe469ebaf1c14ad1b11afb39f900e6f1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fe469ebaf1c14ad1b11afb39f900e6f1 2024-11-16T14:14:59,687 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fe469ebaf1c14ad1b11afb39f900e6f1, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T14:14:59,688 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c646687c72e5cbf577a48e3dcdc6f92b in 962ms, sequenceid=156, compaction requested=false 2024-11-16T14:14:59,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:14:59,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:59,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-16T14:14:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-16T14:14:59,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-16T14:14:59,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1170 sec 2024-11-16T14:14:59,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.1260 sec 2024-11-16T14:14:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T14:14:59,706 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:14:59,706 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:14:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-16T14:14:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:14:59,708 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:14:59,709 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:14:59,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:14:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:14:59,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T14:14:59,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:14:59,863 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T14:14:59,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:14:59,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:59,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:14:59,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:59,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:14:59,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:14:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111627c3b2d3b70941dd833fa2e5275a9cfd_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766498894/Put/seqid=0 2024-11-16T14:14:59,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742001_1177 (size=12304) 2024-11-16T14:15:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:15:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:00,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:00,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766560092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:00,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766560092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:00,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766560098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:00,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766560206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:00,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766560206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:00,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766560206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:00,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:00,279 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111627c3b2d3b70941dd833fa2e5275a9cfd_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111627c3b2d3b70941dd833fa2e5275a9cfd_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:00,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/84dfb22674e24a58abddd0198111bafd, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:00,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/84dfb22674e24a58abddd0198111bafd is 175, key is test_row_0/A:col10/1731766498894/Put/seqid=0 2024-11-16T14:15:00,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742002_1178 (size=31105) 2024-11-16T14:15:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:15:00,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766560416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766560416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766560416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:00,685 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/84dfb22674e24a58abddd0198111bafd 2024-11-16T14:15:00,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/fa8c9ae6534e46168000ff505856bae8 is 50, key is test_row_0/B:col10/1731766498894/Put/seqid=0 2024-11-16T14:15:00,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742003_1179 (size=12151) 2024-11-16T14:15:00,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766560725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:00,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:00,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766560725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766560725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:15:01,099 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/fa8c9ae6534e46168000ff505856bae8 2024-11-16T14:15:01,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/b719a5638c064508b927e9d853f3a9c5 is 50, key is test_row_0/C:col10/1731766498894/Put/seqid=0 2024-11-16T14:15:01,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742004_1180 (size=12151) 2024-11-16T14:15:01,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766561235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:01,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766561235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:01,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766561235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:01,510 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/b719a5638c064508b927e9d853f3a9c5 2024-11-16T14:15:01,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/84dfb22674e24a58abddd0198111bafd as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84dfb22674e24a58abddd0198111bafd 2024-11-16T14:15:01,520 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84dfb22674e24a58abddd0198111bafd, entries=150, sequenceid=173, filesize=30.4 K 2024-11-16T14:15:01,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/fa8c9ae6534e46168000ff505856bae8 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fa8c9ae6534e46168000ff505856bae8 2024-11-16T14:15:01,525 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fa8c9ae6534e46168000ff505856bae8, entries=150, sequenceid=173, filesize=11.9 K 2024-11-16T14:15:01,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/b719a5638c064508b927e9d853f3a9c5 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/b719a5638c064508b927e9d853f3a9c5 2024-11-16T14:15:01,531 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/b719a5638c064508b927e9d853f3a9c5, entries=150, sequenceid=173, filesize=11.9 K 2024-11-16T14:15:01,532 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c646687c72e5cbf577a48e3dcdc6f92b in 1668ms, sequenceid=173, compaction requested=true 2024-11-16T14:15:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-16T14:15:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-16T14:15:01,534 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-16T14:15:01,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8240 sec 2024-11-16T14:15:01,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8290 sec 2024-11-16T14:15:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:15:01,855 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:01,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-16T14:15:01,858 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T14:15:01,859 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:01,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T14:15:02,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T14:15:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:02,012 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T14:15:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:15:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:15:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:15:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:02,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165bfea30e67ca4974aabc00d8f5e37a37_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766500056/Put/seqid=0 2024-11-16T14:15:02,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742005_1181 (size=12304) 2024-11-16T14:15:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T14:15:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:02,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:02,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766562280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:02,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766562281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:02,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:02,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766562281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:02,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766562386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:02,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:02,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766562396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:02,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:02,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766562396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:02,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:02,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:02,440 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165bfea30e67ca4974aabc00d8f5e37a37_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165bfea30e67ca4974aabc00d8f5e37a37_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:02,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/46ccadbc836d4b8c91fe2a2ee02934e0, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:02,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/46ccadbc836d4b8c91fe2a2ee02934e0 is 175, key is test_row_0/A:col10/1731766500056/Put/seqid=0 2024-11-16T14:15:02,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742006_1182 (size=31105) 2024-11-16T14:15:02,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T14:15:02,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766562595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:02,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766562615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:02,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766562616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:02,846 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/46ccadbc836d4b8c91fe2a2ee02934e0 2024-11-16T14:15:02,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/b9af5f4364d6419aafec09624686994a is 50, key is test_row_0/B:col10/1731766500056/Put/seqid=0 2024-11-16T14:15:02,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742007_1183 (size=12151) 2024-11-16T14:15:02,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766562906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:02,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:02,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766562925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766562925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T14:15:03,258 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/b9af5f4364d6419aafec09624686994a 2024-11-16T14:15:03,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/fc6866fd09d54254abf1d2cc7a41a963 is 50, key is test_row_0/C:col10/1731766500056/Put/seqid=0 2024-11-16T14:15:03,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742008_1184 (size=12151) 2024-11-16T14:15:03,274 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/fc6866fd09d54254abf1d2cc7a41a963 2024-11-16T14:15:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/46ccadbc836d4b8c91fe2a2ee02934e0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/46ccadbc836d4b8c91fe2a2ee02934e0 2024-11-16T14:15:03,282 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/46ccadbc836d4b8c91fe2a2ee02934e0, entries=150, sequenceid=188, filesize=30.4 K 2024-11-16T14:15:03,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/b9af5f4364d6419aafec09624686994a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/b9af5f4364d6419aafec09624686994a 2024-11-16T14:15:03,291 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/b9af5f4364d6419aafec09624686994a, entries=150, sequenceid=188, filesize=11.9 K 2024-11-16T14:15:03,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/fc6866fd09d54254abf1d2cc7a41a963 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fc6866fd09d54254abf1d2cc7a41a963 2024-11-16T14:15:03,295 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fc6866fd09d54254abf1d2cc7a41a963, entries=150, sequenceid=188, filesize=11.9 K 2024-11-16T14:15:03,297 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c646687c72e5cbf577a48e3dcdc6f92b in 1285ms, sequenceid=188, compaction requested=true 2024-11-16T14:15:03,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:03,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:03,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-16T14:15:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-16T14:15:03,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-16T14:15:03,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4390 sec 2024-11-16T14:15:03,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4440 sec 2024-11-16T14:15:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:03,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:15:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:15:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:15:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:15:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:03,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116166994df844c422b9545dd6d2f878b59_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766502278/Put/seqid=0 2024-11-16T14:15:03,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742009_1185 (size=14794) 2024-11-16T14:15:03,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766563451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:03,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766563453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:03,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766563454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:03,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:03,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766563565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766563565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:03,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766563566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:03,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766563790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:03,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766563790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766563790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:03,842 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:03,846 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116166994df844c422b9545dd6d2f878b59_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116166994df844c422b9545dd6d2f878b59_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:03,847 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/8a4c69b1202e4e4a847648d276c90789, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:03,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/8a4c69b1202e4e4a847648d276c90789 is 175, key is test_row_0/A:col10/1731766502278/Put/seqid=0 2024-11-16T14:15:03,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742010_1186 (size=39749) 2024-11-16T14:15:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T14:15:04,006 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:04,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-16T14:15:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:15:04,009 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:04,009 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:04,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:04,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766564106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766564106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:04,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766564106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:15:04,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:15:04,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:04,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:04,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:04,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:04,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:04,252 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/8a4c69b1202e4e4a847648d276c90789 2024-11-16T14:15:04,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/a5b07b4f578944a4a37e8253f465b487 is 50, key is test_row_0/B:col10/1731766502278/Put/seqid=0 2024-11-16T14:15:04,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:15:04,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:04,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:04,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742011_1187 (size=12151) 2024-11-16T14:15:04,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:04,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:04,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:04,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:04,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/a5b07b4f578944a4a37e8253f465b487 2024-11-16T14:15:04,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/338a0726bc824c39a33a60da77191188 is 50, key is test_row_0/C:col10/1731766502278/Put/seqid=0 2024-11-16T14:15:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:15:04,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742012_1188 (size=12151) 2024-11-16T14:15:04,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/338a0726bc824c39a33a60da77191188 2024-11-16T14:15:04,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/8a4c69b1202e4e4a847648d276c90789 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8a4c69b1202e4e4a847648d276c90789 2024-11-16T14:15:04,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8a4c69b1202e4e4a847648d276c90789, entries=200, sequenceid=207, filesize=38.8 K 2024-11-16T14:15:04,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/a5b07b4f578944a4a37e8253f465b487 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/a5b07b4f578944a4a37e8253f465b487 2024-11-16T14:15:04,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/a5b07b4f578944a4a37e8253f465b487, entries=150, sequenceid=207, filesize=11.9 K 2024-11-16T14:15:04,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/338a0726bc824c39a33a60da77191188 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/338a0726bc824c39a33a60da77191188 2024-11-16T14:15:04,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/338a0726bc824c39a33a60da77191188, entries=150, sequenceid=207, filesize=11.9 K 2024-11-16T14:15:04,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c646687c72e5cbf577a48e3dcdc6f92b in 918ms, sequenceid=207, compaction requested=true 2024-11-16T14:15:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:04,345 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T14:15:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:04,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:04,346 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T14:15:04,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:04,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:04,347 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T14:15:04,347 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164477 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T14:15:04,347 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/A is initiating minor compaction (all files) 2024-11-16T14:15:04,347 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/B is initiating minor compaction (all files) 2024-11-16T14:15:04,348 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/B in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:04,348 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/A in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:04,348 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/20c1d784715f45c58cf76376323d5faa, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/4c3983059850469ebcd7576eea001546, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fa8c9ae6534e46168000ff505856bae8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/b9af5f4364d6419aafec09624686994a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/a5b07b4f578944a4a37e8253f465b487] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=59.6 K 2024-11-16T14:15:04,348 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/83fcf45f1d3246cbaedc1b91cfb895ef, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/65aaaff06f274e58a78c82daa4c88567, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84dfb22674e24a58abddd0198111bafd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/46ccadbc836d4b8c91fe2a2ee02934e0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8a4c69b1202e4e4a847648d276c90789] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=160.6 K 2024-11-16T14:15:04,348 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:04,348 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/83fcf45f1d3246cbaedc1b91cfb895ef, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/65aaaff06f274e58a78c82daa4c88567, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84dfb22674e24a58abddd0198111bafd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/46ccadbc836d4b8c91fe2a2ee02934e0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8a4c69b1202e4e4a847648d276c90789] 2024-11-16T14:15:04,348 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83fcf45f1d3246cbaedc1b91cfb895ef, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731766496568 2024-11-16T14:15:04,348 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 20c1d784715f45c58cf76376323d5faa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731766496568 2024-11-16T14:15:04,348 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65aaaff06f274e58a78c82daa4c88567, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731766496706 2024-11-16T14:15:04,349 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c3983059850469ebcd7576eea001546, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731766496706 2024-11-16T14:15:04,349 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84dfb22674e24a58abddd0198111bafd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731766498894 2024-11-16T14:15:04,349 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46ccadbc836d4b8c91fe2a2ee02934e0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731766500056 2024-11-16T14:15:04,349 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a4c69b1202e4e4a847648d276c90789, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731766502261 2024-11-16T14:15:04,350 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting fa8c9ae6534e46168000ff505856bae8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731766498894 2024-11-16T14:15:04,350 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting b9af5f4364d6419aafec09624686994a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731766500056 2024-11-16T14:15:04,350 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting a5b07b4f578944a4a37e8253f465b487, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731766502261 2024-11-16T14:15:04,356 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:04,357 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111629f8d9ae495741428761b11ecb413507_c646687c72e5cbf577a48e3dcdc6f92b store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:04,360 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:04,360 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/7f86f75c5b6440949c116c4ec7bfff01 is 50, key is test_row_0/B:col10/1731766502278/Put/seqid=0 2024-11-16T14:15:04,361 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111629f8d9ae495741428761b11ecb413507_c646687c72e5cbf577a48e3dcdc6f92b, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:04,361 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111629f8d9ae495741428761b11ecb413507_c646687c72e5cbf577a48e3dcdc6f92b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:04,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742014_1190 (size=4469) 2024-11-16T14:15:04,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742013_1189 (size=12629) 2024-11-16T14:15:04,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:15:04,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:04,468 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:15:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:15:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:15:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:15:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:04,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169309b2bceaec4dec985777ff98bd62ff_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766503448/Put/seqid=0 2024-11-16T14:15:04,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742015_1191 (size=12304) 2024-11-16T14:15:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:04,482 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169309b2bceaec4dec985777ff98bd62ff_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169309b2bceaec4dec985777ff98bd62ff_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:04,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/40f1059b457e45a5926cb6c10e98966e, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:04,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/40f1059b457e45a5926cb6c10e98966e is 175, key is test_row_0/A:col10/1731766503448/Put/seqid=0 2024-11-16T14:15:04,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742016_1192 (size=31105) 2024-11-16T14:15:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:04,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:15:04,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766564647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:04,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766564648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:04,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766564649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:04,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:04,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766564755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:04,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766564755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:04,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766564755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:04,766 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#A#compaction#159 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:04,767 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/1a2bca9201784ee18472a0a77b0d2c97 is 175, key is test_row_0/A:col10/1731766502278/Put/seqid=0 2024-11-16T14:15:04,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742017_1193 (size=31583) 2024-11-16T14:15:04,775 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/7f86f75c5b6440949c116c4ec7bfff01 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/7f86f75c5b6440949c116c4ec7bfff01 2024-11-16T14:15:04,780 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/B of c646687c72e5cbf577a48e3dcdc6f92b into 7f86f75c5b6440949c116c4ec7bfff01(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:04,780 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:04,780 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/B, priority=11, startTime=1731766504345; duration=0sec 2024-11-16T14:15:04,780 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:04,780 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:B 2024-11-16T14:15:04,780 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T14:15:04,782 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T14:15:04,782 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/C is initiating minor compaction (all files) 2024-11-16T14:15:04,782 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/C in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:04,782 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/045fe9d286c348fdab10eb72cc289cc0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fe469ebaf1c14ad1b11afb39f900e6f1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/b719a5638c064508b927e9d853f3a9c5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fc6866fd09d54254abf1d2cc7a41a963, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/338a0726bc824c39a33a60da77191188] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=59.6 K 2024-11-16T14:15:04,782 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 045fe9d286c348fdab10eb72cc289cc0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731766496568 2024-11-16T14:15:04,783 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting fe469ebaf1c14ad1b11afb39f900e6f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731766496706 2024-11-16T14:15:04,783 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting b719a5638c064508b927e9d853f3a9c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731766498894 2024-11-16T14:15:04,783 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting fc6866fd09d54254abf1d2cc7a41a963, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731766500056 2024-11-16T14:15:04,783 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 338a0726bc824c39a33a60da77191188, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731766502261 2024-11-16T14:15:04,793 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#C#compaction#162 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:04,793 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/ec37f70ec1154e46a9320d2501e6e8c4 is 50, key is test_row_0/C:col10/1731766502278/Put/seqid=0 2024-11-16T14:15:04,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742018_1194 (size=12629) 2024-11-16T14:15:04,888 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/40f1059b457e45a5926cb6c10e98966e 2024-11-16T14:15:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/d2a7631b64464fcea0e0a0fd96fc01cf is 50, key is test_row_0/B:col10/1731766503448/Put/seqid=0 2024-11-16T14:15:04,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742019_1195 (size=12151) 2024-11-16T14:15:04,904 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/d2a7631b64464fcea0e0a0fd96fc01cf 2024-11-16T14:15:04,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/7bd313fe2451430da12194080d88ba84 is 50, key is test_row_0/C:col10/1731766503448/Put/seqid=0 2024-11-16T14:15:04,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742020_1196 (size=12151) 2024-11-16T14:15:04,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:04,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766564965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:04,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:04,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:04,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766564965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:04,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:04,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766564965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:04,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:04,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:04,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:15:05,178 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/1a2bca9201784ee18472a0a77b0d2c97 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/1a2bca9201784ee18472a0a77b0d2c97 2024-11-16T14:15:05,183 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/A of c646687c72e5cbf577a48e3dcdc6f92b into 1a2bca9201784ee18472a0a77b0d2c97(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:05,183 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:05,183 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/A, priority=11, startTime=1731766504345; duration=0sec 2024-11-16T14:15:05,183 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:05,183 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:A 2024-11-16T14:15:05,202 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/ec37f70ec1154e46a9320d2501e6e8c4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ec37f70ec1154e46a9320d2501e6e8c4 2024-11-16T14:15:05,207 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/C of c646687c72e5cbf577a48e3dcdc6f92b into ec37f70ec1154e46a9320d2501e6e8c4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:05,207 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:05,207 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/C, priority=11, startTime=1731766504346; duration=0sec 2024-11-16T14:15:05,207 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:05,207 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:C 2024-11-16T14:15:05,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766565276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:05,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766565276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:05,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766565276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:05,320 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/7bd313fe2451430da12194080d88ba84 2024-11-16T14:15:05,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/40f1059b457e45a5926cb6c10e98966e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/40f1059b457e45a5926cb6c10e98966e 2024-11-16T14:15:05,328 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/40f1059b457e45a5926cb6c10e98966e, entries=150, sequenceid=220, filesize=30.4 K 2024-11-16T14:15:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/d2a7631b64464fcea0e0a0fd96fc01cf as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/d2a7631b64464fcea0e0a0fd96fc01cf 2024-11-16T14:15:05,334 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/d2a7631b64464fcea0e0a0fd96fc01cf, entries=150, sequenceid=220, filesize=11.9 K 2024-11-16T14:15:05,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/7bd313fe2451430da12194080d88ba84 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/7bd313fe2451430da12194080d88ba84 2024-11-16T14:15:05,340 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/7bd313fe2451430da12194080d88ba84, entries=150, sequenceid=220, filesize=11.9 K 2024-11-16T14:15:05,341 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c646687c72e5cbf577a48e3dcdc6f92b in 874ms, sequenceid=220, compaction requested=false 2024-11-16T14:15:05,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:05,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:05,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-16T14:15:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-16T14:15:05,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-16T14:15:05,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3340 sec 2024-11-16T14:15:05,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.3380 sec 2024-11-16T14:15:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:05,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:15:05,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:15:05,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:05,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:15:05,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:05,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:15:05,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:05,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eac6ef1a6f134d6c8cdf05fc3d9c9c04_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766505789/Put/seqid=0 2024-11-16T14:15:05,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742021_1197 (size=17284) 2024-11-16T14:15:05,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766565800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766565808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766565808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766565916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:05,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766565916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766565916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:06,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766566126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:06,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766566126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:06,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766566126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:15:06,155 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:06,156 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-16T14:15:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:15:06,157 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:06,158 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:06,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:06,204 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:06,207 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eac6ef1a6f134d6c8cdf05fc3d9c9c04_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eac6ef1a6f134d6c8cdf05fc3d9c9c04_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:06,208 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/521b7fc896b947f6b622dc7c5a0b2bbc, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:06,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/521b7fc896b947f6b622dc7c5a0b2bbc is 175, key is test_row_0/A:col10/1731766505789/Put/seqid=0 2024-11-16T14:15:06,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742022_1198 (size=48389) 2024-11-16T14:15:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:15:06,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T14:15:06,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:06,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:06,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:06,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766566435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:06,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766566436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:06,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766566436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:06,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T14:15:06,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:06,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:06,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:06,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:15:06,613 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/521b7fc896b947f6b622dc7c5a0b2bbc 2024-11-16T14:15:06,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T14:15:06,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:06,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:06,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:06,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/cf2b1c42193945f19edf9f7dca474529 is 50, key is test_row_0/B:col10/1731766505789/Put/seqid=0 2024-11-16T14:15:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742023_1199 (size=12151) 2024-11-16T14:15:06,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/cf2b1c42193945f19edf9f7dca474529 2024-11-16T14:15:06,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/6b8939d7bca54175a5f8b5dc8b478128 is 50, key is test_row_0/C:col10/1731766505789/Put/seqid=0 2024-11-16T14:15:06,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742024_1200 (size=12151) 2024-11-16T14:15:06,720 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:15:06,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T14:15:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:06,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:15:06,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T14:15:06,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:06,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:06,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:06,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:06,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766566946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:06,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766566946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:06,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766566946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:07,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/6b8939d7bca54175a5f8b5dc8b478128 2024-11-16T14:15:07,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/521b7fc896b947f6b622dc7c5a0b2bbc as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/521b7fc896b947f6b622dc7c5a0b2bbc 2024-11-16T14:15:07,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/521b7fc896b947f6b622dc7c5a0b2bbc, entries=250, sequenceid=245, filesize=47.3 K 2024-11-16T14:15:07,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/cf2b1c42193945f19edf9f7dca474529 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/cf2b1c42193945f19edf9f7dca474529 2024-11-16T14:15:07,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/cf2b1c42193945f19edf9f7dca474529, entries=150, sequenceid=245, filesize=11.9 K 2024-11-16T14:15:07,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/6b8939d7bca54175a5f8b5dc8b478128 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/6b8939d7bca54175a5f8b5dc8b478128 2024-11-16T14:15:07,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T14:15:07,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:07,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:07,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:07,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:07,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:07,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/6b8939d7bca54175a5f8b5dc8b478128, entries=150, sequenceid=245, filesize=11.9 K 2024-11-16T14:15:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:07,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for c646687c72e5cbf577a48e3dcdc6f92b in 1293ms, sequenceid=245, compaction requested=true 2024-11-16T14:15:07,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:07,083 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:07,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:07,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:07,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:07,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:07,083 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:07,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:07,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:07,084 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111077 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:07,084 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/A is initiating minor compaction (all files) 2024-11-16T14:15:07,084 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/A in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:07,084 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:07,084 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/1a2bca9201784ee18472a0a77b0d2c97, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/40f1059b457e45a5926cb6c10e98966e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/521b7fc896b947f6b622dc7c5a0b2bbc] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=108.5 K 2024-11-16T14:15:07,084 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/B is initiating minor compaction (all files) 2024-11-16T14:15:07,084 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:07,084 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/B in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:07,084 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/1a2bca9201784ee18472a0a77b0d2c97, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/40f1059b457e45a5926cb6c10e98966e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/521b7fc896b947f6b622dc7c5a0b2bbc] 2024-11-16T14:15:07,085 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/7f86f75c5b6440949c116c4ec7bfff01, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/d2a7631b64464fcea0e0a0fd96fc01cf, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/cf2b1c42193945f19edf9f7dca474529] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=36.1 K 2024-11-16T14:15:07,085 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a2bca9201784ee18472a0a77b0d2c97, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731766502261 2024-11-16T14:15:07,085 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f86f75c5b6440949c116c4ec7bfff01, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731766502261 2024-11-16T14:15:07,085 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting d2a7631b64464fcea0e0a0fd96fc01cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731766503441 2024-11-16T14:15:07,085 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40f1059b457e45a5926cb6c10e98966e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731766503441 2024-11-16T14:15:07,085 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2b1c42193945f19edf9f7dca474529, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731766504639 2024-11-16T14:15:07,085 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 521b7fc896b947f6b622dc7c5a0b2bbc, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731766504639 2024-11-16T14:15:07,094 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:07,096 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:07,096 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/86eb693c19a141929c040267ba4997d7 is 50, key is test_row_0/B:col10/1731766505789/Put/seqid=0 2024-11-16T14:15:07,096 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116eee28a312aac470581e487615d0c5542_c646687c72e5cbf577a48e3dcdc6f92b store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:07,099 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116eee28a312aac470581e487615d0c5542_c646687c72e5cbf577a48e3dcdc6f92b, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:07,099 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eee28a312aac470581e487615d0c5542_c646687c72e5cbf577a48e3dcdc6f92b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:07,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742025_1201 (size=12731) 2024-11-16T14:15:07,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742026_1202 (size=4469) 2024-11-16T14:15:07,110 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/86eb693c19a141929c040267ba4997d7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/86eb693c19a141929c040267ba4997d7 2024-11-16T14:15:07,115 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/B of c646687c72e5cbf577a48e3dcdc6f92b into 86eb693c19a141929c040267ba4997d7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:07,115 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:07,115 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/B, priority=13, startTime=1731766507083; duration=0sec 2024-11-16T14:15:07,115 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:07,115 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:B 2024-11-16T14:15:07,115 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:07,116 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:07,116 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/C is initiating minor compaction (all files) 2024-11-16T14:15:07,116 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/C in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:07,116 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ec37f70ec1154e46a9320d2501e6e8c4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/7bd313fe2451430da12194080d88ba84, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/6b8939d7bca54175a5f8b5dc8b478128] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=36.1 K 2024-11-16T14:15:07,117 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ec37f70ec1154e46a9320d2501e6e8c4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731766502261 2024-11-16T14:15:07,117 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd313fe2451430da12194080d88ba84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731766503441 2024-11-16T14:15:07,117 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b8939d7bca54175a5f8b5dc8b478128, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731766504639 2024-11-16T14:15:07,124 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:07,125 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/8e1c1523bd984c9895060e4c146067d0 is 50, key is test_row_0/C:col10/1731766505789/Put/seqid=0 2024-11-16T14:15:07,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742027_1203 (size=12731) 2024-11-16T14:15:07,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T14:15:07,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:07,234 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:15:07,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:15:07,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:07,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:15:07,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:07,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:15:07,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161e713d9889a54cb48927f53feae2bf6c_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766505804/Put/seqid=0 2024-11-16T14:15:07,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742028_1204 (size=12304) 2024-11-16T14:15:07,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:07,287 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161e713d9889a54cb48927f53feae2bf6c_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161e713d9889a54cb48927f53feae2bf6c_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:07,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/84ddc45c40e045d284c8fd1f2780f086, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:15:07,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/84ddc45c40e045d284c8fd1f2780f086 is 175, key is test_row_0/A:col10/1731766505804/Put/seqid=0 2024-11-16T14:15:07,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742029_1205 (size=31105) 2024-11-16T14:15:07,508 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#A#compaction#169 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:07,509 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/9a5917e45fff465091a86bef3387228c is 175, key is test_row_0/A:col10/1731766505789/Put/seqid=0 2024-11-16T14:15:07,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742030_1206 (size=31685) 2024-11-16T14:15:07,526 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/9a5917e45fff465091a86bef3387228c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/9a5917e45fff465091a86bef3387228c 2024-11-16T14:15:07,532 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/A of c646687c72e5cbf577a48e3dcdc6f92b into 9a5917e45fff465091a86bef3387228c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:07,532 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:07,532 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/A, priority=13, startTime=1731766507082; duration=0sec 2024-11-16T14:15:07,533 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:07,533 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:A 2024-11-16T14:15:07,537 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/8e1c1523bd984c9895060e4c146067d0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/8e1c1523bd984c9895060e4c146067d0 2024-11-16T14:15:07,542 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/C of c646687c72e5cbf577a48e3dcdc6f92b into 8e1c1523bd984c9895060e4c146067d0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:07,542 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:07,542 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/C, priority=13, startTime=1731766507083; duration=0sec 2024-11-16T14:15:07,543 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:07,543 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:C 2024-11-16T14:15:07,730 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/84ddc45c40e045d284c8fd1f2780f086 2024-11-16T14:15:07,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/d713d41470e241c18752124355e5a637 is 50, key is test_row_0/B:col10/1731766505804/Put/seqid=0 2024-11-16T14:15:07,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742031_1207 (size=12151) 2024-11-16T14:15:07,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:07,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:08,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766568047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766508018, max timestamp=1731766508052Num uniques 859; 2024-11-16T14:15:08,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766568078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766568079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766568166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766568186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766568196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,200 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/d713d41470e241c18752124355e5a637 2024-11-16T14:15:08,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/e210633e4ce94041b747a85a82cda7d4 is 50, key is test_row_0/C:col10/1731766505804/Put/seqid=0 2024-11-16T14:15:08,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742032_1208 (size=12151) 2024-11-16T14:15:08,244 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/e210633e4ce94041b747a85a82cda7d4 2024-11-16T14:15:08,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/84ddc45c40e045d284c8fd1f2780f086 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84ddc45c40e045d284c8fd1f2780f086 2024-11-16T14:15:08,260 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84ddc45c40e045d284c8fd1f2780f086, entries=150, sequenceid=256, filesize=30.4 K 2024-11-16T14:15:08,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/d713d41470e241c18752124355e5a637 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/d713d41470e241c18752124355e5a637 2024-11-16T14:15:08,277 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/d713d41470e241c18752124355e5a637, entries=150, sequenceid=256, filesize=11.9 K 2024-11-16T14:15:08,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/e210633e4ce94041b747a85a82cda7d4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/e210633e4ce94041b747a85a82cda7d4 2024-11-16T14:15:08,305 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/e210633e4ce94041b747a85a82cda7d4, entries=150, sequenceid=256, filesize=11.9 K 2024-11-16T14:15:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:15:08,307 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for c646687c72e5cbf577a48e3dcdc6f92b in 1073ms, sequenceid=256, compaction requested=false 2024-11-16T14:15:08,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:08,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:08,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-16T14:15:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-16T14:15:08,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-16T14:15:08,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1510 sec 2024-11-16T14:15:08,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.1780 sec 2024-11-16T14:15:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:08,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=145.36 KB heapSize=426.56 KB 2024-11-16T14:15:08,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:15:08,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:08,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:15:08,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:08,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:15:08,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:08,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766568409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111604f4b23cce314e16b5936193f2ef3176_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766508379/Put/seqid=0 2024-11-16T14:15:08,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766568413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766568415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742033_1209 (size=14994) 2024-11-16T14:15:08,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766568526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766568536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766568536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766568735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766568756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766568756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:08,901 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:08,905 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111604f4b23cce314e16b5936193f2ef3176_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111604f4b23cce314e16b5936193f2ef3176_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:08,905 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/e637d49858244ffb98e35ef474e609cd, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:08,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/e637d49858244ffb98e35ef474e609cd is 175, key is test_row_0/A:col10/1731766508379/Put/seqid=0 2024-11-16T14:15:08,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742034_1210 (size=39949) 2024-11-16T14:15:09,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766569056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:09,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:09,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766569066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766569067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:09,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:09,310 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/e637d49858244ffb98e35ef474e609cd 2024-11-16T14:15:09,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/6916e6eb65644137a0e89c0ee0deafd4 is 50, key is test_row_0/B:col10/1731766508379/Put/seqid=0 2024-11-16T14:15:09,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742035_1211 (size=12301) 2024-11-16T14:15:09,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58106 deadline: 1731766569545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:09,547 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in c646687c72e5cbf577a48e3dcdc6f92b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18315 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:15:09,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:09,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58140 deadline: 1731766569555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:09,558 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in c646687c72e5cbf577a48e3dcdc6f92b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18330 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:15:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:09,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766569576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:09,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766569576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766569585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:09,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/6916e6eb65644137a0e89c0ee0deafd4 2024-11-16T14:15:09,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/1777c5acf01b4788b3e27a833319db3b is 50, key is test_row_0/C:col10/1731766508379/Put/seqid=0 2024-11-16T14:15:09,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742036_1212 (size=12301) 2024-11-16T14:15:09,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/1777c5acf01b4788b3e27a833319db3b 2024-11-16T14:15:09,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/e637d49858244ffb98e35ef474e609cd as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/e637d49858244ffb98e35ef474e609cd 2024-11-16T14:15:09,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/e637d49858244ffb98e35ef474e609cd, entries=200, sequenceid=287, filesize=39.0 K 2024-11-16T14:15:09,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/6916e6eb65644137a0e89c0ee0deafd4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/6916e6eb65644137a0e89c0ee0deafd4 2024-11-16T14:15:09,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/6916e6eb65644137a0e89c0ee0deafd4, entries=150, sequenceid=287, filesize=12.0 K 2024-11-16T14:15:09,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/1777c5acf01b4788b3e27a833319db3b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/1777c5acf01b4788b3e27a833319db3b 2024-11-16T14:15:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/1777c5acf01b4788b3e27a833319db3b, entries=150, sequenceid=287, filesize=12.0 K 2024-11-16T14:15:09,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~426.52 KB/436752, currentSize=33.54 KB/34350 for c646687c72e5cbf577a48e3dcdc6f92b in 1425ms, sequenceid=287, compaction requested=true 2024-11-16T14:15:09,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:09,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:09,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:09,808 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:09,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:09,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:09,808 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:09,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:09,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:09,810 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:09,810 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:09,810 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/B is initiating minor compaction (all files) 2024-11-16T14:15:09,810 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/A is initiating minor compaction (all files) 2024-11-16T14:15:09,810 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/B in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:09,810 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/A in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:09,810 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/86eb693c19a141929c040267ba4997d7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/d713d41470e241c18752124355e5a637, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/6916e6eb65644137a0e89c0ee0deafd4] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=36.3 K 2024-11-16T14:15:09,810 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/9a5917e45fff465091a86bef3387228c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84ddc45c40e045d284c8fd1f2780f086, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/e637d49858244ffb98e35ef474e609cd] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=100.3 K 2024-11-16T14:15:09,810 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:09,810 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/9a5917e45fff465091a86bef3387228c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84ddc45c40e045d284c8fd1f2780f086, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/e637d49858244ffb98e35ef474e609cd] 2024-11-16T14:15:09,811 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a5917e45fff465091a86bef3387228c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731766504639 2024-11-16T14:15:09,812 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 86eb693c19a141929c040267ba4997d7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731766504639 2024-11-16T14:15:09,812 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84ddc45c40e045d284c8fd1f2780f086, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731766505792 2024-11-16T14:15:09,812 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting d713d41470e241c18752124355e5a637, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731766505792 2024-11-16T14:15:09,812 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e637d49858244ffb98e35ef474e609cd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731766508067 2024-11-16T14:15:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,813 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6916e6eb65644137a0e89c0ee0deafd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731766508067 2024-11-16T14:15:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,837 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:09,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,844 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#B#compaction#178 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,844 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/86f94d27c5f54f7d90df3dc052e8a734 is 50, key is test_row_0/B:col10/1731766508379/Put/seqid=0 2024-11-16T14:15:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,857 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116b61418a3d73647e6abd640521fe53c6b_c646687c72e5cbf577a48e3dcdc6f92b store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:09,860 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116b61418a3d73647e6abd640521fe53c6b_c646687c72e5cbf577a48e3dcdc6f92b, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:09,860 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b61418a3d73647e6abd640521fe53c6b_c646687c72e5cbf577a48e3dcdc6f92b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742037_1213 (size=12983) 2024-11-16T14:15:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,881 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/86f94d27c5f54f7d90df3dc052e8a734 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/86f94d27c5f54f7d90df3dc052e8a734 2024-11-16T14:15:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,888 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/B of c646687c72e5cbf577a48e3dcdc6f92b into 86f94d27c5f54f7d90df3dc052e8a734(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:09,888 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:09,888 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/B, priority=13, startTime=1731766509808; duration=0sec 2024-11-16T14:15:09,889 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:09,889 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:B 2024-11-16T14:15:09,889 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,892 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:09,892 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/C is initiating minor compaction (all files) 2024-11-16T14:15:09,892 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/C in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:09,892 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/8e1c1523bd984c9895060e4c146067d0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/e210633e4ce94041b747a85a82cda7d4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/1777c5acf01b4788b3e27a833319db3b] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=36.3 K 2024-11-16T14:15:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,893 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e1c1523bd984c9895060e4c146067d0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731766504639 2024-11-16T14:15:09,893 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting e210633e4ce94041b747a85a82cda7d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731766505792 2024-11-16T14:15:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,894 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1777c5acf01b4788b3e27a833319db3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731766508067 2024-11-16T14:15:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742038_1214 (size=4469) 2024-11-16T14:15:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,920 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,920 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/edfca21bb8c14a9ea5d6c51f7020e841 is 50, key is test_row_0/C:col10/1731766508379/Put/seqid=0 2024-11-16T14:15:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,922 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#A#compaction#177 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,922 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/ca5e173322d54368b716741b36863569 is 175, key is test_row_0/A:col10/1731766508379/Put/seqid=0 2024-11-16T14:15:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742039_1215 (size=31937) 2024-11-16T14:15:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,941 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/ca5e173322d54368b716741b36863569 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/ca5e173322d54368b716741b36863569 2024-11-16T14:15:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,951 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/A of c646687c72e5cbf577a48e3dcdc6f92b into ca5e173322d54368b716741b36863569(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:09,951 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:09,951 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/A, priority=13, startTime=1731766509808; duration=0sec 2024-11-16T14:15:09,951 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:09,951 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:A 2024-11-16T14:15:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742040_1216 (size=12983) 2024-11-16T14:15:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:15:10,325 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-16T14:15:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,330 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T14:15:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,332 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,388 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/edfca21bb8c14a9ea5d6c51f7020e841 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/edfca21bb8c14a9ea5d6c51f7020e841 2024-11-16T14:15:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,395 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/C of c646687c72e5cbf577a48e3dcdc6f92b into edfca21bb8c14a9ea5d6c51f7020e841(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,395 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:10,395 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/C, priority=13, startTime=1731766509808; duration=0sec 2024-11-16T14:15:10,395 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:10,395 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:C 2024-11-16T14:15:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T14:15:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T14:15:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,485 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T14:15:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:15:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:15:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:15:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f270574424774ebbaa0cb766e624a9ec_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766508406/Put/seqid=0 2024-11-16T14:15:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742041_1217 (size=9914) 2024-11-16T14:15:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:10,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. as already flushing 2024-11-16T14:15:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766510627, max timestamp=1731766510641Num uniques 859; 2024-11-16T14:15:10,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766510627, max timestamp=1731766510641Num uniques 859; 2024-11-16T14:15:10,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:15:10,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:15:10,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766510627, max timestamp=1731766510641Num uniques 859; 2024-11-16T14:15:10,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:15:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T14:15:10,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766570688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:10,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766570689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:10,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766570689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:10,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766570806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:10,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766570806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:10,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766570806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:10,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:10,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:10,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:10,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:10,903 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f270574424774ebbaa0cb766e624a9ec_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f270574424774ebbaa0cb766e624a9ec_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:10,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/8acd14f6b51341fcb47af6ab244601f3, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:10,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/8acd14f6b51341fcb47af6ab244601f3 is 175, key is test_row_0/A:col10/1731766508406/Put/seqid=0 2024-11-16T14:15:10,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742042_1218 (size=22561) 2024-11-16T14:15:10,955 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/8acd14f6b51341fcb47af6ab244601f3 2024-11-16T14:15:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T14:15:10,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/89f202f483704c83808a903763ebbff9 is 50, key is test_row_0/B:col10/1731766508406/Put/seqid=0 2024-11-16T14:15:11,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742043_1219 (size=9857) 2024-11-16T14:15:11,007 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/89f202f483704c83808a903763ebbff9 2024-11-16T14:15:11,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/36f4e0171e70409cb149bd44f311cfeb is 50, key is test_row_0/C:col10/1731766508406/Put/seqid=0 2024-11-16T14:15:11,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742044_1220 (size=9857) 2024-11-16T14:15:11,032 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/36f4e0171e70409cb149bd44f311cfeb 2024-11-16T14:15:11,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1731766571025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:11,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:11,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58048 deadline: 1731766571026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58118 deadline: 1731766571025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:11,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/8acd14f6b51341fcb47af6ab244601f3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8acd14f6b51341fcb47af6ab244601f3 2024-11-16T14:15:11,056 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8acd14f6b51341fcb47af6ab244601f3, entries=100, sequenceid=298, filesize=22.0 K 2024-11-16T14:15:11,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/89f202f483704c83808a903763ebbff9 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/89f202f483704c83808a903763ebbff9 2024-11-16T14:15:11,062 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/89f202f483704c83808a903763ebbff9, entries=100, sequenceid=298, filesize=9.6 K 2024-11-16T14:15:11,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/36f4e0171e70409cb149bd44f311cfeb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/36f4e0171e70409cb149bd44f311cfeb 2024-11-16T14:15:11,085 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-16T14:15:11,085 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:11,085 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,085 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/36f4e0171e70409cb149bd44f311cfeb, entries=100, sequenceid=298, filesize=9.6 K 2024-11-16T14:15:11,085 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:11,086 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for c646687c72e5cbf577a48e3dcdc6f92b in 601ms, sequenceid=298, compaction requested=false 2024-11-16T14:15:11,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:11,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:11,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-16T14:15:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-16T14:15:11,089 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-16T14:15:11,089 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:11,089 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,089 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,089 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:11,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-16T14:15:11,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 756 msec 2024-11-16T14:15:11,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 764 msec 2024-11-16T14:15:11,094 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-16T14:15:11,094 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:11,094 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,094 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,094 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:11,099 INFO [Thread-679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-679. 2024-11-16T14:15:11,099 DEBUG [Thread-679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:11,099 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,099 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:11,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=154.31 KB heapSize=436.08 KB 2024-11-16T14:15:11,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:15:11,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T14:15:11,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:15:11,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T14:15:11,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:15:11,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T14:15:11,339 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-16T14:15:11,339 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:11,339 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,339 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:11,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ce4bef11f78b46f8a57c4781d4e710b3_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_0/A:col10/1731766510668/Put/seqid=0 2024-11-16T14:15:11,347 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-16T14:15:11,347 INFO [Thread-666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-666. 2024-11-16T14:15:11,347 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:11,347 DEBUG [Thread-666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:11,347 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,347 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,347 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,347 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:11,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:11,348 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:11,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742045_1221 (size=12454) 2024-11-16T14:15:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T14:15:11,466 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:11,750 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:11,755 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ce4bef11f78b46f8a57c4781d4e710b3_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ce4bef11f78b46f8a57c4781d4e710b3_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:11,756 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/095a663b86d24c9ebd35c4d75c400210, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:11,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/095a663b86d24c9ebd35c4d75c400210 is 175, key is test_row_0/A:col10/1731766510668/Put/seqid=0 2024-11-16T14:15:11,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742046_1222 (size=31255) 2024-11-16T14:15:12,161 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/095a663b86d24c9ebd35c4d75c400210 2024-11-16T14:15:12,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/84180f9e54824355a613f408d0c7bd73 is 50, key is test_row_0/B:col10/1731766510668/Put/seqid=0 2024-11-16T14:15:12,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742047_1223 (size=12301) 2024-11-16T14:15:12,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/84180f9e54824355a613f408d0c7bd73 2024-11-16T14:15:12,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/6f24e1cc8dc9410dab27d910e182302b is 50, key is test_row_0/C:col10/1731766510668/Put/seqid=0 2024-11-16T14:15:12,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742048_1224 (size=12301) 2024-11-16T14:15:12,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/6f24e1cc8dc9410dab27d910e182302b 2024-11-16T14:15:12,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/095a663b86d24c9ebd35c4d75c400210 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/095a663b86d24c9ebd35c4d75c400210 2024-11-16T14:15:12,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/095a663b86d24c9ebd35c4d75c400210, entries=150, sequenceid=336, filesize=30.5 K 2024-11-16T14:15:12,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/84180f9e54824355a613f408d0c7bd73 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/84180f9e54824355a613f408d0c7bd73 2024-11-16T14:15:12,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/84180f9e54824355a613f408d0c7bd73, entries=150, sequenceid=336, filesize=12.0 K 2024-11-16T14:15:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/6f24e1cc8dc9410dab27d910e182302b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/6f24e1cc8dc9410dab27d910e182302b 2024-11-16T14:15:12,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/6f24e1cc8dc9410dab27d910e182302b, entries=150, sequenceid=336, filesize=12.0 K 2024-11-16T14:15:12,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=13.42 KB/13740 for c646687c72e5cbf577a48e3dcdc6f92b in 1287ms, sequenceid=336, compaction requested=true 2024-11-16T14:15:12,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:12,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:12,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:12,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:12,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:12,626 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:12,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c646687c72e5cbf577a48e3dcdc6f92b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:12,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:12,626 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:12,627 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:12,627 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:12,627 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/A is initiating minor compaction (all files) 2024-11-16T14:15:12,627 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/B is initiating minor compaction (all files) 2024-11-16T14:15:12,627 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/A in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:12,627 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/B in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:12,627 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/ca5e173322d54368b716741b36863569, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8acd14f6b51341fcb47af6ab244601f3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/095a663b86d24c9ebd35c4d75c400210] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=83.7 K 2024-11-16T14:15:12,627 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/86f94d27c5f54f7d90df3dc052e8a734, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/89f202f483704c83808a903763ebbff9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/84180f9e54824355a613f408d0c7bd73] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=34.3 K 2024-11-16T14:15:12,627 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:12,627 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/ca5e173322d54368b716741b36863569, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8acd14f6b51341fcb47af6ab244601f3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/095a663b86d24c9ebd35c4d75c400210] 2024-11-16T14:15:12,627 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 86f94d27c5f54f7d90df3dc052e8a734, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731766508067 2024-11-16T14:15:12,627 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca5e173322d54368b716741b36863569, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731766508067 2024-11-16T14:15:12,628 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8acd14f6b51341fcb47af6ab244601f3, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731766508406 2024-11-16T14:15:12,628 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 89f202f483704c83808a903763ebbff9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731766508406 2024-11-16T14:15:12,628 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 095a663b86d24c9ebd35c4d75c400210, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731766510668 2024-11-16T14:15:12,628 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 84180f9e54824355a613f408d0c7bd73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731766510668 2024-11-16T14:15:12,637 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#B#compaction#186 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:12,637 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:12,638 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/57881249ce17426f9ba49acdf4d0792a is 50, key is test_row_0/B:col10/1731766510668/Put/seqid=0 2024-11-16T14:15:12,639 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111613e1dcff70464ef19419e504c2193cf5_c646687c72e5cbf577a48e3dcdc6f92b store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:12,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742049_1225 (size=13085) 2024-11-16T14:15:12,653 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/57881249ce17426f9ba49acdf4d0792a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/57881249ce17426f9ba49acdf4d0792a 2024-11-16T14:15:12,657 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111613e1dcff70464ef19419e504c2193cf5_c646687c72e5cbf577a48e3dcdc6f92b, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:12,657 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111613e1dcff70464ef19419e504c2193cf5_c646687c72e5cbf577a48e3dcdc6f92b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:12,659 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/B of c646687c72e5cbf577a48e3dcdc6f92b into 57881249ce17426f9ba49acdf4d0792a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:12,659 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:12,659 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/B, priority=13, startTime=1731766512626; duration=0sec 2024-11-16T14:15:12,659 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:12,659 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:B 2024-11-16T14:15:12,659 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:12,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742050_1226 (size=4469) 2024-11-16T14:15:12,661 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:12,662 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): c646687c72e5cbf577a48e3dcdc6f92b/C is initiating minor compaction (all files) 2024-11-16T14:15:12,662 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c646687c72e5cbf577a48e3dcdc6f92b/C in TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:12,662 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/edfca21bb8c14a9ea5d6c51f7020e841, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/36f4e0171e70409cb149bd44f311cfeb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/6f24e1cc8dc9410dab27d910e182302b] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp, totalSize=34.3 K 2024-11-16T14:15:12,662 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting edfca21bb8c14a9ea5d6c51f7020e841, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731766508067 2024-11-16T14:15:12,663 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 36f4e0171e70409cb149bd44f311cfeb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731766508406 2024-11-16T14:15:12,663 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f24e1cc8dc9410dab27d910e182302b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731766510668 2024-11-16T14:15:12,673 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#C#compaction#188 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:12,673 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/39533e81738144869012306d09b39f0e is 50, key is test_row_0/C:col10/1731766510668/Put/seqid=0 2024-11-16T14:15:12,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742051_1227 (size=13085) 2024-11-16T14:15:13,062 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c646687c72e5cbf577a48e3dcdc6f92b#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:13,062 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/b0e51a05b2394baa974dd90b50670daa is 175, key is test_row_0/A:col10/1731766510668/Put/seqid=0 2024-11-16T14:15:13,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742052_1228 (size=32039) 2024-11-16T14:15:13,082 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/39533e81738144869012306d09b39f0e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/39533e81738144869012306d09b39f0e 2024-11-16T14:15:13,087 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/C of c646687c72e5cbf577a48e3dcdc6f92b into 39533e81738144869012306d09b39f0e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:13,087 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:13,087 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/C, priority=13, startTime=1731766512626; duration=0sec 2024-11-16T14:15:13,087 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:13,088 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:C 2024-11-16T14:15:13,471 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/b0e51a05b2394baa974dd90b50670daa as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/b0e51a05b2394baa974dd90b50670daa 2024-11-16T14:15:13,477 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c646687c72e5cbf577a48e3dcdc6f92b/A of c646687c72e5cbf577a48e3dcdc6f92b into b0e51a05b2394baa974dd90b50670daa(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:13,477 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c646687c72e5cbf577a48e3dcdc6f92b: 2024-11-16T14:15:13,477 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., storeName=c646687c72e5cbf577a48e3dcdc6f92b/A, priority=13, startTime=1731766512626; duration=0sec 2024-11-16T14:15:13,477 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:13,477 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c646687c72e5cbf577a48e3dcdc6f92b:A 2024-11-16T14:15:19,608 INFO [Thread-670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-670. 2024-11-16T14:15:19,608 DEBUG [Thread-670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:19,608 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:19,609 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:19,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:19,627 INFO [Thread-668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-668. 2024-11-16T14:15:19,627 DEBUG [Thread-668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:19,627 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:19,627 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:19,627 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3108 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3032 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1236 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3705 rows 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1233 2024-11-16T14:15:19,627 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3698 rows 2024-11-16T14:15:19,627 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:15:19,627 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:15:19,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:19,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:19,628 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:19,628 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T14:15:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T14:15:19,631 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766519631"}]},"ts":"1731766519631"} 2024-11-16T14:15:19,632 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T14:15:19,632 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T14:15:19,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:15:19,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, UNASSIGN}] 2024-11-16T14:15:19,634 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, UNASSIGN 2024-11-16T14:15:19,635 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=c646687c72e5cbf577a48e3dcdc6f92b, regionState=CLOSING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:19,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, UNASSIGN because future has completed 2024-11-16T14:15:19,636 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:15:19,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:15:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T14:15:19,789 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:19,789 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:15:19,789 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing c646687c72e5cbf577a48e3dcdc6f92b, disabling compactions & flushes 2024-11-16T14:15:19,789 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:19,789 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:19,789 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. after waiting 0 ms 2024-11-16T14:15:19,789 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:19,790 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing c646687c72e5cbf577a48e3dcdc6f92b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-16T14:15:19,790 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=A 2024-11-16T14:15:19,790 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:19,790 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=B 2024-11-16T14:15:19,790 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:19,790 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c646687c72e5cbf577a48e3dcdc6f92b, store=C 2024-11-16T14:15:19,790 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:19,798 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f733dbee14334de2b777e7c79cf049c1_c646687c72e5cbf577a48e3dcdc6f92b is 50, key is test_row_2/A:col10/1731766519625/Put/seqid=0 2024-11-16T14:15:19,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742053_1229 (size=7374) 2024-11-16T14:15:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T14:15:20,203 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:20,209 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f733dbee14334de2b777e7c79cf049c1_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f733dbee14334de2b777e7c79cf049c1_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:20,209 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/6dc86e61f26a436d854a01e8e76815a1, store: [table=TestAcidGuarantees family=A region=c646687c72e5cbf577a48e3dcdc6f92b] 2024-11-16T14:15:20,210 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/6dc86e61f26a436d854a01e8e76815a1 is 175, key is test_row_2/A:col10/1731766519625/Put/seqid=0 2024-11-16T14:15:20,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742054_1230 (size=13865) 2024-11-16T14:15:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T14:15:20,614 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/6dc86e61f26a436d854a01e8e76815a1 2024-11-16T14:15:20,623 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/889f87441af845c4b3c80b8b83d024ff is 50, key is test_row_2/B:col10/1731766519625/Put/seqid=0 2024-11-16T14:15:20,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742055_1231 (size=7415) 2024-11-16T14:15:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T14:15:21,028 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/889f87441af845c4b3c80b8b83d024ff 2024-11-16T14:15:21,041 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/09a5c6b493ca4b4097efacf4424e25d2 is 50, key is test_row_2/C:col10/1731766519625/Put/seqid=0 2024-11-16T14:15:21,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742056_1232 (size=7415) 2024-11-16T14:15:21,445 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/09a5c6b493ca4b4097efacf4424e25d2 2024-11-16T14:15:21,453 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/A/6dc86e61f26a436d854a01e8e76815a1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/6dc86e61f26a436d854a01e8e76815a1 2024-11-16T14:15:21,460 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/6dc86e61f26a436d854a01e8e76815a1, entries=50, sequenceid=346, filesize=13.5 K 2024-11-16T14:15:21,461 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/B/889f87441af845c4b3c80b8b83d024ff as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/889f87441af845c4b3c80b8b83d024ff 2024-11-16T14:15:21,466 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/889f87441af845c4b3c80b8b83d024ff, entries=50, sequenceid=346, filesize=7.2 K 2024-11-16T14:15:21,467 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/.tmp/C/09a5c6b493ca4b4097efacf4424e25d2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/09a5c6b493ca4b4097efacf4424e25d2 2024-11-16T14:15:21,472 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/09a5c6b493ca4b4097efacf4424e25d2, entries=50, sequenceid=346, filesize=7.2 K 2024-11-16T14:15:21,473 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c646687c72e5cbf577a48e3dcdc6f92b in 1684ms, sequenceid=346, compaction requested=false 2024-11-16T14:15:21,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/82f66dd8d97a421997618d45216d9c95, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/c2322f6fc02d4cf29885fefab9eba526, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/0c45988af30240198d892b3ff299d004, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/985b3d9545d84e49b75b9e341d63fc13, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8dff098fc27b4dee9e4ff9e47f1e945c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/b4697348334e4c32976888d9e1f270be, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/a73b3377de9c4b6aaf6ac2b9809c53a4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/34a4b6903b3a43769437509e07c83a44, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/5247f3df9f3d4092a2f63b0e5ae9d5ab, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/83fcf45f1d3246cbaedc1b91cfb895ef, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/65aaaff06f274e58a78c82daa4c88567, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84dfb22674e24a58abddd0198111bafd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/46ccadbc836d4b8c91fe2a2ee02934e0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8a4c69b1202e4e4a847648d276c90789, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/1a2bca9201784ee18472a0a77b0d2c97, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/40f1059b457e45a5926cb6c10e98966e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/521b7fc896b947f6b622dc7c5a0b2bbc, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/9a5917e45fff465091a86bef3387228c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84ddc45c40e045d284c8fd1f2780f086, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/e637d49858244ffb98e35ef474e609cd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/ca5e173322d54368b716741b36863569, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8acd14f6b51341fcb47af6ab244601f3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/095a663b86d24c9ebd35c4d75c400210] to archive 2024-11-16T14:15:21,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:15:21,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/82f66dd8d97a421997618d45216d9c95 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/82f66dd8d97a421997618d45216d9c95 2024-11-16T14:15:21,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/c2322f6fc02d4cf29885fefab9eba526 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/c2322f6fc02d4cf29885fefab9eba526 2024-11-16T14:15:21,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/0c45988af30240198d892b3ff299d004 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/0c45988af30240198d892b3ff299d004 2024-11-16T14:15:21,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/985b3d9545d84e49b75b9e341d63fc13 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/985b3d9545d84e49b75b9e341d63fc13 2024-11-16T14:15:21,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8dff098fc27b4dee9e4ff9e47f1e945c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8dff098fc27b4dee9e4ff9e47f1e945c 2024-11-16T14:15:21,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/b4697348334e4c32976888d9e1f270be to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/b4697348334e4c32976888d9e1f270be 2024-11-16T14:15:21,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/a73b3377de9c4b6aaf6ac2b9809c53a4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/a73b3377de9c4b6aaf6ac2b9809c53a4 2024-11-16T14:15:21,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/34a4b6903b3a43769437509e07c83a44 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/34a4b6903b3a43769437509e07c83a44 2024-11-16T14:15:21,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/5247f3df9f3d4092a2f63b0e5ae9d5ab to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/5247f3df9f3d4092a2f63b0e5ae9d5ab 2024-11-16T14:15:21,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/83fcf45f1d3246cbaedc1b91cfb895ef to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/83fcf45f1d3246cbaedc1b91cfb895ef 2024-11-16T14:15:21,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/65aaaff06f274e58a78c82daa4c88567 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/65aaaff06f274e58a78c82daa4c88567 2024-11-16T14:15:21,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84dfb22674e24a58abddd0198111bafd to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84dfb22674e24a58abddd0198111bafd 2024-11-16T14:15:21,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/46ccadbc836d4b8c91fe2a2ee02934e0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/46ccadbc836d4b8c91fe2a2ee02934e0 2024-11-16T14:15:21,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8a4c69b1202e4e4a847648d276c90789 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8a4c69b1202e4e4a847648d276c90789 2024-11-16T14:15:21,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/1a2bca9201784ee18472a0a77b0d2c97 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/1a2bca9201784ee18472a0a77b0d2c97 2024-11-16T14:15:21,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/40f1059b457e45a5926cb6c10e98966e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/40f1059b457e45a5926cb6c10e98966e 2024-11-16T14:15:21,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/521b7fc896b947f6b622dc7c5a0b2bbc to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/521b7fc896b947f6b622dc7c5a0b2bbc 2024-11-16T14:15:21,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/9a5917e45fff465091a86bef3387228c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/9a5917e45fff465091a86bef3387228c 2024-11-16T14:15:21,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84ddc45c40e045d284c8fd1f2780f086 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/84ddc45c40e045d284c8fd1f2780f086 2024-11-16T14:15:21,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/e637d49858244ffb98e35ef474e609cd to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/e637d49858244ffb98e35ef474e609cd 2024-11-16T14:15:21,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/ca5e173322d54368b716741b36863569 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/ca5e173322d54368b716741b36863569 2024-11-16T14:15:21,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8acd14f6b51341fcb47af6ab244601f3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/8acd14f6b51341fcb47af6ab244601f3 2024-11-16T14:15:21,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/095a663b86d24c9ebd35c4d75c400210 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/095a663b86d24c9ebd35c4d75c400210 2024-11-16T14:15:21,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/35a90528b7a84f439c08940b0a7b9a76, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/491fa816fd3a48d9be731df8dda97d37, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/1617d7dc9b014a6d80222dd6f230376b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/28d8122a8ffe4bd88909c8e288b0006b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fa72a3f26338493fb9d6024c620d1ecf, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/ba6bd23393224b17a921b3b530d7be26, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/0b46e832037c44cf85c7cc02fe426fd6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/9bdbdc8bd2dd4aeea056380c9b745769, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/20c1d784715f45c58cf76376323d5faa, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fc512898c72c4a70a604c02b8295fb03, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/4c3983059850469ebcd7576eea001546, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fa8c9ae6534e46168000ff505856bae8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/b9af5f4364d6419aafec09624686994a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/7f86f75c5b6440949c116c4ec7bfff01, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/a5b07b4f578944a4a37e8253f465b487, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/d2a7631b64464fcea0e0a0fd96fc01cf, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/86eb693c19a141929c040267ba4997d7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/cf2b1c42193945f19edf9f7dca474529, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/d713d41470e241c18752124355e5a637, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/86f94d27c5f54f7d90df3dc052e8a734, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/6916e6eb65644137a0e89c0ee0deafd4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/89f202f483704c83808a903763ebbff9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/84180f9e54824355a613f408d0c7bd73] to archive 2024-11-16T14:15:21,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:15:21,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/35a90528b7a84f439c08940b0a7b9a76 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/35a90528b7a84f439c08940b0a7b9a76 2024-11-16T14:15:21,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/491fa816fd3a48d9be731df8dda97d37 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/491fa816fd3a48d9be731df8dda97d37 2024-11-16T14:15:21,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/1617d7dc9b014a6d80222dd6f230376b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/1617d7dc9b014a6d80222dd6f230376b 2024-11-16T14:15:21,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/28d8122a8ffe4bd88909c8e288b0006b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/28d8122a8ffe4bd88909c8e288b0006b 2024-11-16T14:15:21,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fa72a3f26338493fb9d6024c620d1ecf to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fa72a3f26338493fb9d6024c620d1ecf 2024-11-16T14:15:21,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/ba6bd23393224b17a921b3b530d7be26 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/ba6bd23393224b17a921b3b530d7be26 2024-11-16T14:15:21,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/0b46e832037c44cf85c7cc02fe426fd6 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/0b46e832037c44cf85c7cc02fe426fd6 2024-11-16T14:15:21,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/9bdbdc8bd2dd4aeea056380c9b745769 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/9bdbdc8bd2dd4aeea056380c9b745769 2024-11-16T14:15:21,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/20c1d784715f45c58cf76376323d5faa to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/20c1d784715f45c58cf76376323d5faa 2024-11-16T14:15:21,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fc512898c72c4a70a604c02b8295fb03 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fc512898c72c4a70a604c02b8295fb03 2024-11-16T14:15:21,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/4c3983059850469ebcd7576eea001546 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/4c3983059850469ebcd7576eea001546 2024-11-16T14:15:21,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fa8c9ae6534e46168000ff505856bae8 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/fa8c9ae6534e46168000ff505856bae8 2024-11-16T14:15:21,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/b9af5f4364d6419aafec09624686994a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/b9af5f4364d6419aafec09624686994a 2024-11-16T14:15:21,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/7f86f75c5b6440949c116c4ec7bfff01 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/7f86f75c5b6440949c116c4ec7bfff01 2024-11-16T14:15:21,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/a5b07b4f578944a4a37e8253f465b487 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/a5b07b4f578944a4a37e8253f465b487 2024-11-16T14:15:21,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/d2a7631b64464fcea0e0a0fd96fc01cf to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/d2a7631b64464fcea0e0a0fd96fc01cf 2024-11-16T14:15:21,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/86eb693c19a141929c040267ba4997d7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/86eb693c19a141929c040267ba4997d7 2024-11-16T14:15:21,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/cf2b1c42193945f19edf9f7dca474529 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/cf2b1c42193945f19edf9f7dca474529 2024-11-16T14:15:21,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/d713d41470e241c18752124355e5a637 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/d713d41470e241c18752124355e5a637 2024-11-16T14:15:21,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/86f94d27c5f54f7d90df3dc052e8a734 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/86f94d27c5f54f7d90df3dc052e8a734 2024-11-16T14:15:21,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/6916e6eb65644137a0e89c0ee0deafd4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/6916e6eb65644137a0e89c0ee0deafd4 2024-11-16T14:15:21,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/89f202f483704c83808a903763ebbff9 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/89f202f483704c83808a903763ebbff9 2024-11-16T14:15:21,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/84180f9e54824355a613f408d0c7bd73 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/84180f9e54824355a613f408d0c7bd73 2024-11-16T14:15:21,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/f31b3b73ec1e4df6bda0762a66eb194b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ad746a6b45a64b0e8b286b2419206864, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/22f26026b5ff4e039c15972cf4a1c006, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/991b3640dddc4390acf402f86c65d1ae, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/b9a2ebe3b056405397c9a6909e85a714, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ae73c26b9921414aa46ea45b9812c12a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/f0417496a4854b3e8a354bdf29fbad44, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fb8242262aa645b3821eedbec0ac4e05, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/045fe9d286c348fdab10eb72cc289cc0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/959c09de00464c78a2d801cb48c94a21, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fe469ebaf1c14ad1b11afb39f900e6f1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/b719a5638c064508b927e9d853f3a9c5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fc6866fd09d54254abf1d2cc7a41a963, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ec37f70ec1154e46a9320d2501e6e8c4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/338a0726bc824c39a33a60da77191188, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/7bd313fe2451430da12194080d88ba84, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/8e1c1523bd984c9895060e4c146067d0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/6b8939d7bca54175a5f8b5dc8b478128, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/e210633e4ce94041b747a85a82cda7d4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/edfca21bb8c14a9ea5d6c51f7020e841, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/1777c5acf01b4788b3e27a833319db3b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/36f4e0171e70409cb149bd44f311cfeb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/6f24e1cc8dc9410dab27d910e182302b] to archive 2024-11-16T14:15:21,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:15:21,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/f31b3b73ec1e4df6bda0762a66eb194b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/f31b3b73ec1e4df6bda0762a66eb194b 2024-11-16T14:15:21,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ad746a6b45a64b0e8b286b2419206864 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ad746a6b45a64b0e8b286b2419206864 2024-11-16T14:15:21,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/22f26026b5ff4e039c15972cf4a1c006 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/22f26026b5ff4e039c15972cf4a1c006 2024-11-16T14:15:21,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/991b3640dddc4390acf402f86c65d1ae to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/991b3640dddc4390acf402f86c65d1ae 2024-11-16T14:15:21,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/b9a2ebe3b056405397c9a6909e85a714 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/b9a2ebe3b056405397c9a6909e85a714 2024-11-16T14:15:21,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ae73c26b9921414aa46ea45b9812c12a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ae73c26b9921414aa46ea45b9812c12a 2024-11-16T14:15:21,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/f0417496a4854b3e8a354bdf29fbad44 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/f0417496a4854b3e8a354bdf29fbad44 2024-11-16T14:15:21,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fb8242262aa645b3821eedbec0ac4e05 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fb8242262aa645b3821eedbec0ac4e05 2024-11-16T14:15:21,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/045fe9d286c348fdab10eb72cc289cc0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/045fe9d286c348fdab10eb72cc289cc0 2024-11-16T14:15:21,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/959c09de00464c78a2d801cb48c94a21 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/959c09de00464c78a2d801cb48c94a21 2024-11-16T14:15:21,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fe469ebaf1c14ad1b11afb39f900e6f1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fe469ebaf1c14ad1b11afb39f900e6f1 2024-11-16T14:15:21,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/b719a5638c064508b927e9d853f3a9c5 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/b719a5638c064508b927e9d853f3a9c5 2024-11-16T14:15:21,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fc6866fd09d54254abf1d2cc7a41a963 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/fc6866fd09d54254abf1d2cc7a41a963 2024-11-16T14:15:21,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ec37f70ec1154e46a9320d2501e6e8c4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/ec37f70ec1154e46a9320d2501e6e8c4 2024-11-16T14:15:21,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/338a0726bc824c39a33a60da77191188 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/338a0726bc824c39a33a60da77191188 2024-11-16T14:15:21,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/7bd313fe2451430da12194080d88ba84 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/7bd313fe2451430da12194080d88ba84 2024-11-16T14:15:21,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/8e1c1523bd984c9895060e4c146067d0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/8e1c1523bd984c9895060e4c146067d0 2024-11-16T14:15:21,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/6b8939d7bca54175a5f8b5dc8b478128 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/6b8939d7bca54175a5f8b5dc8b478128 2024-11-16T14:15:21,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/e210633e4ce94041b747a85a82cda7d4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/e210633e4ce94041b747a85a82cda7d4 2024-11-16T14:15:21,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/edfca21bb8c14a9ea5d6c51f7020e841 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/edfca21bb8c14a9ea5d6c51f7020e841 2024-11-16T14:15:21,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/1777c5acf01b4788b3e27a833319db3b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/1777c5acf01b4788b3e27a833319db3b 2024-11-16T14:15:21,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/36f4e0171e70409cb149bd44f311cfeb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/36f4e0171e70409cb149bd44f311cfeb 2024-11-16T14:15:21,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/6f24e1cc8dc9410dab27d910e182302b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/6f24e1cc8dc9410dab27d910e182302b 2024-11-16T14:15:21,550 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/recovered.edits/349.seqid, newMaxSeqId=349, maxSeqId=4 2024-11-16T14:15:21,551 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. 2024-11-16T14:15:21,551 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for c646687c72e5cbf577a48e3dcdc6f92b: Waiting for close lock at 1731766519789Running coprocessor pre-close hooks at 1731766519789Disabling compacts and flushes for region at 1731766519789Disabling writes for close at 1731766519789Obtaining lock to block concurrent updates at 1731766519790 (+1 ms)Preparing flush snapshotting stores in c646687c72e5cbf577a48e3dcdc6f92b at 1731766519790Finished memstore snapshotting TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731766519790Flushing stores of TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b. at 1731766519791 (+1 ms)Flushing c646687c72e5cbf577a48e3dcdc6f92b/A: creating writer at 1731766519791Flushing mob file c646687c72e5cbf577a48e3dcdc6f92b/A: appending metadata at 1731766519797 (+6 ms)Flushing mob file c646687c72e5cbf577a48e3dcdc6f92b/A: closing flushed file at 1731766519797Flushing c646687c72e5cbf577a48e3dcdc6f92b/A: appending metadata at 1731766520210 (+413 ms)Flushing c646687c72e5cbf577a48e3dcdc6f92b/A: closing flushed file at 1731766520210Flushing c646687c72e5cbf577a48e3dcdc6f92b/B: creating writer at 1731766520619 (+409 ms)Flushing c646687c72e5cbf577a48e3dcdc6f92b/B: appending metadata at 1731766520622 (+3 ms)Flushing c646687c72e5cbf577a48e3dcdc6f92b/B: closing flushed file at 1731766520622Flushing c646687c72e5cbf577a48e3dcdc6f92b/C: creating writer at 1731766521036 (+414 ms)Flushing c646687c72e5cbf577a48e3dcdc6f92b/C: appending metadata at 1731766521040 (+4 ms)Flushing c646687c72e5cbf577a48e3dcdc6f92b/C: closing flushed file at 1731766521040Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61ffe1b6: reopening flushed file at 1731766521451 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a7d03a7: reopening flushed file at 1731766521460 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72996045: reopening flushed file at 1731766521466 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c646687c72e5cbf577a48e3dcdc6f92b in 1684ms, sequenceid=346, compaction requested=false at 1731766521473 (+7 ms)Writing region close event to WAL at 1731766521547 (+74 ms)Running coprocessor post-close hooks at 1731766521551 (+4 ms)Closed at 1731766521551 2024-11-16T14:15:21,553 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,553 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=c646687c72e5cbf577a48e3dcdc6f92b, regionState=CLOSED 2024-11-16T14:15:21,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:15:21,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-16T14:15:21,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure c646687c72e5cbf577a48e3dcdc6f92b, server=04e93f2b5fad,41317,1731766448342 in 1.9190 sec 2024-11-16T14:15:21,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-16T14:15:21,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c646687c72e5cbf577a48e3dcdc6f92b, UNASSIGN in 1.9220 sec 2024-11-16T14:15:21,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-16T14:15:21,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9260 sec 2024-11-16T14:15:21,562 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766521562"}]},"ts":"1731766521562"} 2024-11-16T14:15:21,564 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T14:15:21,564 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T14:15:21,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9370 sec 2024-11-16T14:15:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T14:15:21,776 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:21,776 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T14:15:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:21,778 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T14:15:21,779 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:21,782 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,785 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/recovered.edits] 2024-11-16T14:15:21,788 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/6dc86e61f26a436d854a01e8e76815a1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/6dc86e61f26a436d854a01e8e76815a1 2024-11-16T14:15:21,789 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/b0e51a05b2394baa974dd90b50670daa to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/A/b0e51a05b2394baa974dd90b50670daa 2024-11-16T14:15:21,792 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/57881249ce17426f9ba49acdf4d0792a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/57881249ce17426f9ba49acdf4d0792a 2024-11-16T14:15:21,793 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/889f87441af845c4b3c80b8b83d024ff to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/B/889f87441af845c4b3c80b8b83d024ff 2024-11-16T14:15:21,796 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/09a5c6b493ca4b4097efacf4424e25d2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/09a5c6b493ca4b4097efacf4424e25d2 2024-11-16T14:15:21,798 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/39533e81738144869012306d09b39f0e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/C/39533e81738144869012306d09b39f0e 2024-11-16T14:15:21,802 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/recovered.edits/349.seqid to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b/recovered.edits/349.seqid 2024-11-16T14:15:21,802 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,802 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T14:15:21,803 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T14:15:21,804 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T14:15:21,808 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111604f4b23cce314e16b5936193f2ef3176_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111604f4b23cce314e16b5936193f2ef3176_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,809 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116166994df844c422b9545dd6d2f878b59_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116166994df844c422b9545dd6d2f878b59_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,810 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161e713d9889a54cb48927f53feae2bf6c_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161e713d9889a54cb48927f53feae2bf6c_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,811 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161fb5d54a60c84b809ab902bc2a34a3cf_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161fb5d54a60c84b809ab902bc2a34a3cf_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,812 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111627a024b92fef494faf12514c040d7227_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111627a024b92fef494faf12514c040d7227_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,813 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111627c3b2d3b70941dd833fa2e5275a9cfd_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111627c3b2d3b70941dd833fa2e5275a9cfd_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,814 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165bfea30e67ca4974aabc00d8f5e37a37_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165bfea30e67ca4974aabc00d8f5e37a37_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,815 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169309b2bceaec4dec985777ff98bd62ff_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169309b2bceaec4dec985777ff98bd62ff_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,815 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ae57150320ad41a9beed393a66963b2e_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ae57150320ad41a9beed393a66963b2e_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,816 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cde74164fc3d4159b75f609a194faa45_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cde74164fc3d4159b75f609a194faa45_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,817 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ce4bef11f78b46f8a57c4781d4e710b3_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ce4bef11f78b46f8a57c4781d4e710b3_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,818 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d81211871d9b47548af5840a704e0660_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d81211871d9b47548af5840a704e0660_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,819 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d86fb0be13be4edeb613d5df4d7fd994_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d86fb0be13be4edeb613d5df4d7fd994_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,820 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eac6ef1a6f134d6c8cdf05fc3d9c9c04_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eac6ef1a6f134d6c8cdf05fc3d9c9c04_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,821 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f270574424774ebbaa0cb766e624a9ec_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f270574424774ebbaa0cb766e624a9ec_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,822 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f35455774e49478487bc17b86934b190_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f35455774e49478487bc17b86934b190_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,823 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f733dbee14334de2b777e7c79cf049c1_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f733dbee14334de2b777e7c79cf049c1_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,824 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fa336c492ea1434e9d506d7461dd83c9_c646687c72e5cbf577a48e3dcdc6f92b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fa336c492ea1434e9d506d7461dd83c9_c646687c72e5cbf577a48e3dcdc6f92b 2024-11-16T14:15:21,825 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T14:15:21,827 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:21,829 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T14:15:21,831 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T14:15:21,832 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:21,832 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T14:15:21,832 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731766521832"}]},"ts":"9223372036854775807"} 2024-11-16T14:15:21,834 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T14:15:21,834 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c646687c72e5cbf577a48e3dcdc6f92b, NAME => 'TestAcidGuarantees,,1731766487879.c646687c72e5cbf577a48e3dcdc6f92b.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T14:15:21,834 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T14:15:21,834 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731766521834"}]},"ts":"9223372036854775807"} 2024-11-16T14:15:21,835 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T14:15:21,836 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:21,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-16T14:15:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T14:15:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T14:15:21,886 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:21,895 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 250), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 300) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4783 (was 4191) - AvailableMemoryMB LEAK? - 2024-11-16T14:15:21,904 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=4782 2024-11-16T14:15:21,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:15:21,906 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:15:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:21,908 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:15:21,908 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:21,908 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-16T14:15:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T14:15:21,909 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:15:21,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742057_1233 (size=1061) 2024-11-16T14:15:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T14:15:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T14:15:22,317 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc 2024-11-16T14:15:22,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742058_1234 (size=53) 2024-11-16T14:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T14:15:22,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:15:22,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5bda05ae08ffa563beaf32369a8bf2d2, disabling compactions & flushes 2024-11-16T14:15:22,725 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:22,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:22,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. after waiting 0 ms 2024-11-16T14:15:22,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:22,725 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:22,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5bda05ae08ffa563beaf32369a8bf2d2: Waiting for close lock at 1731766522725Disabling compacts and flushes for region at 1731766522725Disabling writes for close at 1731766522725Writing region close event to WAL at 1731766522725Closed at 1731766522725 2024-11-16T14:15:22,726 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:15:22,726 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731766522726"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731766522726"}]},"ts":"1731766522726"} 2024-11-16T14:15:22,728 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:15:22,729 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:15:22,730 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766522729"}]},"ts":"1731766522729"} 2024-11-16T14:15:22,731 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T14:15:22,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bda05ae08ffa563beaf32369a8bf2d2, ASSIGN}] 2024-11-16T14:15:22,733 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bda05ae08ffa563beaf32369a8bf2d2, ASSIGN 2024-11-16T14:15:22,734 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bda05ae08ffa563beaf32369a8bf2d2, ASSIGN; state=OFFLINE, location=04e93f2b5fad,41317,1731766448342; forceNewPlan=false, retain=false 2024-11-16T14:15:22,885 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5bda05ae08ffa563beaf32369a8bf2d2, regionState=OPENING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:22,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bda05ae08ffa563beaf32369a8bf2d2, ASSIGN because future has completed 2024-11-16T14:15:22,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:15:23,041 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:23,041 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:15:23,041 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,041 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:15:23,042 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,042 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,043 INFO [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,044 INFO [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:15:23,044 INFO [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bda05ae08ffa563beaf32369a8bf2d2 columnFamilyName A 2024-11-16T14:15:23,044 DEBUG [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:23,045 INFO [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] regionserver.HStore(327): Store=5bda05ae08ffa563beaf32369a8bf2d2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:15:23,045 INFO [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T14:15:23,046 INFO [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:15:23,046 INFO [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bda05ae08ffa563beaf32369a8bf2d2 columnFamilyName B 2024-11-16T14:15:23,046 DEBUG [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:23,046 INFO [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] regionserver.HStore(327): Store=5bda05ae08ffa563beaf32369a8bf2d2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:15:23,047 INFO [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,048 INFO [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:15:23,048 INFO [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bda05ae08ffa563beaf32369a8bf2d2 columnFamilyName C 2024-11-16T14:15:23,048 DEBUG [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:23,048 INFO [StoreOpener-5bda05ae08ffa563beaf32369a8bf2d2-1 {}] regionserver.HStore(327): Store=5bda05ae08ffa563beaf32369a8bf2d2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:15:23,049 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:23,049 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,049 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,050 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,051 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,051 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,051 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:15:23,052 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,054 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:15:23,055 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 5bda05ae08ffa563beaf32369a8bf2d2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73746186, jitterRate=0.09890380501747131}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:15:23,055 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:23,055 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 5bda05ae08ffa563beaf32369a8bf2d2: Running coprocessor pre-open hook at 1731766523042Writing region info on filesystem at 1731766523042Initializing all the Stores at 1731766523043 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766523043Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766523043Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766523043Cleaning up temporary data from old regions at 1731766523051 (+8 ms)Running coprocessor post-open hooks at 1731766523055 (+4 ms)Region opened successfully at 1731766523055 2024-11-16T14:15:23,056 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., pid=63, masterSystemTime=1731766523038 2024-11-16T14:15:23,058 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:23,058 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:23,058 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5bda05ae08ffa563beaf32369a8bf2d2, regionState=OPEN, openSeqNum=2, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:23,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:15:23,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-16T14:15:23,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 in 174 msec 2024-11-16T14:15:23,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-16T14:15:23,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bda05ae08ffa563beaf32369a8bf2d2, ASSIGN in 329 msec 2024-11-16T14:15:23,063 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:15:23,063 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766523063"}]},"ts":"1731766523063"} 2024-11-16T14:15:23,064 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T14:15:23,065 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:15:23,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-16T14:15:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T14:15:24,056 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:24,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8da64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:24,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:24,058 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:24,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:24,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f191663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:24,060 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a1e1c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,061 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:24,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:24,065 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:15:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 04e93f2b5fad,39231,1731766447537 2024-11-16T14:15:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@159f1899 2024-11-16T14:15:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:15:24,067 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:15:24,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,068 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:24,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:24,069 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:24,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:24,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:24,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f76cc87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:24,071 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:24,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f56a5a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:24,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:24,073 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1adc65cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:24,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:24,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:24,075 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:24,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3452156e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,076 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:24,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:24,077 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f0188c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:24,078 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d93323c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:24,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:24,081 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5849c095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:24,083 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e63149b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:24,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:24,086 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:24,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:24,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7272a6ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:24,088 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5e79e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:24,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:24,090 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2901b21e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:24,091 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71fc80c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,092 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:24,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:24,093 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36aabf1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:24,094 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c065b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,095 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:24,095 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:24,096 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac1f30b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:24,097 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465c3d70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,098 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:24,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:24,099 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc00957, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:24,100 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40ccbad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:24,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:24,101 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c42075e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:24,102 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da87d7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:24,103 DEBUG [Thread-1102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:24,103 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:24,103 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:24,104 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:24,104 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:24,104 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:24,105 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:24,105 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:24,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:24,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:24,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:24,106 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:24,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:24,106 DEBUG [Thread-1100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:24,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-16T14:15:24,107 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:24,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:15:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:24,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:15:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:24,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:24,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:15:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:15:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:15:24,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:24,114 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:15:24,116 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:24,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:15:24,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:15:24,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:15:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:15:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:24,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:15:24,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:24,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/7ad5cf7c3e854ff2a5e5a3cb70ef6a86 is 50, key is test_row_0/A:col10/1731766524126/Put/seqid=0 2024-11-16T14:15:24,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766584162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766584163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766584168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766584163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766584172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742059_1235 (size=12001) 2024-11-16T14:15:24,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:15:24,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:15:24,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:24,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:24,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:24,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766584276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766584276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766584276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766584277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766584286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:15:24,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:15:24,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:24,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:24,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:24,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766584486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766584486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766584486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766584486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766584497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:15:24,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:24,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:24,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:24,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/7ad5cf7c3e854ff2a5e5a3cb70ef6a86 2024-11-16T14:15:24,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/515d7c98d82a4eab9ec99d35aa0c8a41 is 50, key is test_row_0/B:col10/1731766524126/Put/seqid=0 2024-11-16T14:15:24,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742060_1236 (size=12001) 2024-11-16T14:15:24,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:15:24,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:24,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:15:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766584796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766584796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766584796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766584796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766584805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:24,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:15:24,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:24,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:24,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:24,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:15:25,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:25,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:25,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:25,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/515d7c98d82a4eab9ec99d35aa0c8a41 2024-11-16T14:15:25,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/7adee86731284014ae3ea88bcae66148 is 50, key is test_row_0/C:col10/1731766524126/Put/seqid=0 2024-11-16T14:15:25,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742061_1237 (size=12001) 2024-11-16T14:15:25,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:15:25,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:25,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:25,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:25,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:15:25,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:25,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766585305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:25,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766585305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:25,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766585306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:25,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766585306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:25,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766585316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:25,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:15:25,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:25,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:25,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:25,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/7adee86731284014ae3ea88bcae66148 2024-11-16T14:15:25,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:15:25,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:25,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:25,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:25,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:25,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/7ad5cf7c3e854ff2a5e5a3cb70ef6a86 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/7ad5cf7c3e854ff2a5e5a3cb70ef6a86 2024-11-16T14:15:25,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/7ad5cf7c3e854ff2a5e5a3cb70ef6a86, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T14:15:25,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/515d7c98d82a4eab9ec99d35aa0c8a41 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/515d7c98d82a4eab9ec99d35aa0c8a41 2024-11-16T14:15:25,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/515d7c98d82a4eab9ec99d35aa0c8a41, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T14:15:25,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/7adee86731284014ae3ea88bcae66148 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/7adee86731284014ae3ea88bcae66148 2024-11-16T14:15:25,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/7adee86731284014ae3ea88bcae66148, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T14:15:25,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5bda05ae08ffa563beaf32369a8bf2d2 in 1404ms, sequenceid=13, compaction requested=false 2024-11-16T14:15:25,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:25,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:15:25,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:25,659 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:15:25,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:25,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:25,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:25,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:25,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:25,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:25,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/d2fc574c643145419faaa39d6d522606 is 50, key is test_row_0/A:col10/1731766524170/Put/seqid=0 2024-11-16T14:15:25,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742062_1238 (size=12001) 2024-11-16T14:15:26,070 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/d2fc574c643145419faaa39d6d522606 2024-11-16T14:15:26,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/bedc50e1e5b440b0b14341652da48bd1 is 50, key is test_row_0/B:col10/1731766524170/Put/seqid=0 2024-11-16T14:15:26,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742063_1239 (size=12001) 2024-11-16T14:15:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:15:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:26,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:26,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766586325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766586326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766586326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766586326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,330 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in 5bda05ae08ffa563beaf32369a8bf2d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:15:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,330 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in 5bda05ae08ffa563beaf32369a8bf2d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:15:26,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766586327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766586436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766586436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766586436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,540 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/bedc50e1e5b440b0b14341652da48bd1 2024-11-16T14:15:26,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/11a00d6e555d47f28aeb2e3fe5d0115c is 50, key is test_row_0/C:col10/1731766524170/Put/seqid=0 2024-11-16T14:15:26,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742064_1240 (size=12001) 2024-11-16T14:15:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766586646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766586646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766586646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766586955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766586955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766586956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:26,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:26,980 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/11a00d6e555d47f28aeb2e3fe5d0115c 2024-11-16T14:15:26,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/d2fc574c643145419faaa39d6d522606 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/d2fc574c643145419faaa39d6d522606 2024-11-16T14:15:26,988 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/d2fc574c643145419faaa39d6d522606, entries=150, sequenceid=33, filesize=11.7 K 2024-11-16T14:15:26,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/bedc50e1e5b440b0b14341652da48bd1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/bedc50e1e5b440b0b14341652da48bd1 2024-11-16T14:15:26,994 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/bedc50e1e5b440b0b14341652da48bd1, entries=150, sequenceid=33, filesize=11.7 K 2024-11-16T14:15:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/11a00d6e555d47f28aeb2e3fe5d0115c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/11a00d6e555d47f28aeb2e3fe5d0115c 2024-11-16T14:15:26,999 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/11a00d6e555d47f28aeb2e3fe5d0115c, entries=150, sequenceid=33, filesize=11.7 K 2024-11-16T14:15:27,000 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5bda05ae08ffa563beaf32369a8bf2d2 in 1341ms, sequenceid=33, compaction requested=false 2024-11-16T14:15:27,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:27,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:27,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-16T14:15:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-16T14:15:27,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-16T14:15:27,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8860 sec 2024-11-16T14:15:27,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.9010 sec 2024-11-16T14:15:27,078 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:15:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:27,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:15:27,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:27,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:27,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:27,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:27,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:27,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:27,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/aadd48cd40434afe92cd28396ed0fa57 is 50, key is test_row_0/A:col10/1731766526325/Put/seqid=0 2024-11-16T14:15:27,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742065_1241 (size=12001) 2024-11-16T14:15:27,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/aadd48cd40434afe92cd28396ed0fa57 2024-11-16T14:15:27,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/0d5d32c1aead437e88235f3549bd320d is 50, key is test_row_0/B:col10/1731766526325/Put/seqid=0 2024-11-16T14:15:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766587496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:27,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766587499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:27,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:27,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766587501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:27,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742066_1242 (size=12001) 2024-11-16T14:15:27,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/0d5d32c1aead437e88235f3549bd320d 2024-11-16T14:15:27,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/e4914ed20f1f4b29a1a74ec9cc078bbb is 50, key is test_row_0/C:col10/1731766526325/Put/seqid=0 2024-11-16T14:15:27,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742067_1243 (size=12001) 2024-11-16T14:15:27,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766587606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:27,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:27,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766587616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:27,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766587616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:27,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766587816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:27,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766587825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:27,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:27,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766587825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:27,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:27,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:27,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:27,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/e4914ed20f1f4b29a1a74ec9cc078bbb 2024-11-16T14:15:27,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/aadd48cd40434afe92cd28396ed0fa57 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/aadd48cd40434afe92cd28396ed0fa57 2024-11-16T14:15:27,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/aadd48cd40434afe92cd28396ed0fa57, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T14:15:27,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/0d5d32c1aead437e88235f3549bd320d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0d5d32c1aead437e88235f3549bd320d 2024-11-16T14:15:27,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0d5d32c1aead437e88235f3549bd320d, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T14:15:27,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/e4914ed20f1f4b29a1a74ec9cc078bbb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/e4914ed20f1f4b29a1a74ec9cc078bbb 2024-11-16T14:15:27,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/e4914ed20f1f4b29a1a74ec9cc078bbb, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T14:15:27,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5bda05ae08ffa563beaf32369a8bf2d2 in 530ms, sequenceid=46, compaction requested=true 2024-11-16T14:15:27,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:27,998 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:27,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:27,999 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:27,999 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/A is initiating minor compaction (all files) 2024-11-16T14:15:27,999 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/A in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:28,000 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/7ad5cf7c3e854ff2a5e5a3cb70ef6a86, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/d2fc574c643145419faaa39d6d522606, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/aadd48cd40434afe92cd28396ed0fa57] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=35.2 K 2024-11-16T14:15:28,000 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ad5cf7c3e854ff2a5e5a3cb70ef6a86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731766524125 2024-11-16T14:15:28,001 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:28,001 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2fc574c643145419faaa39d6d522606, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731766524161 2024-11-16T14:15:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:28,003 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting aadd48cd40434afe92cd28396ed0fa57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731766526322 2024-11-16T14:15:28,004 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:28,004 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/B is initiating minor compaction (all files) 2024-11-16T14:15:28,004 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/B in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:28,004 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/515d7c98d82a4eab9ec99d35aa0c8a41, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/bedc50e1e5b440b0b14341652da48bd1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0d5d32c1aead437e88235f3549bd320d] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=35.2 K 2024-11-16T14:15:28,005 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 515d7c98d82a4eab9ec99d35aa0c8a41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731766524125 2024-11-16T14:15:28,006 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting bedc50e1e5b440b0b14341652da48bd1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731766524161 2024-11-16T14:15:28,006 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d5d32c1aead437e88235f3549bd320d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731766526322 2024-11-16T14:15:28,025 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:28,025 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/3200bd77247947cbb43d1c99b5b3da8c is 50, key is test_row_0/B:col10/1731766526325/Put/seqid=0 2024-11-16T14:15:28,028 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#A#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:28,028 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/2f42ef8fbd0d474695b9e6fc33ba2a1e is 50, key is test_row_0/A:col10/1731766526325/Put/seqid=0 2024-11-16T14:15:28,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742068_1244 (size=12104) 2024-11-16T14:15:28,078 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/3200bd77247947cbb43d1c99b5b3da8c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/3200bd77247947cbb43d1c99b5b3da8c 2024-11-16T14:15:28,090 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/B of 5bda05ae08ffa563beaf32369a8bf2d2 into 3200bd77247947cbb43d1c99b5b3da8c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:28,090 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:28,090 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/B, priority=13, startTime=1731766528001; duration=0sec 2024-11-16T14:15:28,090 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:28,090 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:B 2024-11-16T14:15:28,091 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:28,092 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:28,093 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/C is initiating minor compaction (all files) 2024-11-16T14:15:28,093 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/C in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:28,093 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/7adee86731284014ae3ea88bcae66148, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/11a00d6e555d47f28aeb2e3fe5d0115c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/e4914ed20f1f4b29a1a74ec9cc078bbb] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=35.2 K 2024-11-16T14:15:28,094 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7adee86731284014ae3ea88bcae66148, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731766524125 2024-11-16T14:15:28,095 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 11a00d6e555d47f28aeb2e3fe5d0115c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731766524161 2024-11-16T14:15:28,096 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting e4914ed20f1f4b29a1a74ec9cc078bbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731766526322 2024-11-16T14:15:28,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742069_1245 (size=12104) 2024-11-16T14:15:28,116 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/2f42ef8fbd0d474695b9e6fc33ba2a1e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/2f42ef8fbd0d474695b9e6fc33ba2a1e 2024-11-16T14:15:28,126 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/A of 5bda05ae08ffa563beaf32369a8bf2d2 into 2f42ef8fbd0d474695b9e6fc33ba2a1e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:28,126 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:28,127 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/A, priority=13, startTime=1731766527997; duration=0sec 2024-11-16T14:15:28,127 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:28,127 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:A 2024-11-16T14:15:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:28,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:15:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:28,135 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:28,136 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/acfdba2062944cef8bb8cbec0adf48d3 is 50, key is test_row_0/C:col10/1731766526325/Put/seqid=0 2024-11-16T14:15:28,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/8b63648d02f046c595055cab0133fb1f is 50, key is test_row_0/A:col10/1731766527496/Put/seqid=0 2024-11-16T14:15:28,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742071_1247 (size=14341) 2024-11-16T14:15:28,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742070_1246 (size=12104) 2024-11-16T14:15:28,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766588159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766588162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766588162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,182 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/acfdba2062944cef8bb8cbec0adf48d3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/acfdba2062944cef8bb8cbec0adf48d3 2024-11-16T14:15:28,194 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/C of 5bda05ae08ffa563beaf32369a8bf2d2 into acfdba2062944cef8bb8cbec0adf48d3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:28,194 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:28,194 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/C, priority=13, startTime=1731766528002; duration=0sec 2024-11-16T14:15:28,194 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:28,194 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:C 2024-11-16T14:15:28,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766588275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766588276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766588276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:15:28,288 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:28,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-16T14:15:28,292 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T14:15:28,293 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:28,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:28,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766588346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,351 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in 5bda05ae08ffa563beaf32369a8bf2d2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:15:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766588356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,361 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in 5bda05ae08ffa563beaf32369a8bf2d2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:15:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T14:15:28,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T14:15:28,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:28,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:28,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:28,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:28,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:28,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766588486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766588486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766588486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/8b63648d02f046c595055cab0133fb1f 2024-11-16T14:15:28,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/c5069cd0846846c8bd9307c8d1e15792 is 50, key is test_row_0/B:col10/1731766527496/Put/seqid=0 2024-11-16T14:15:28,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T14:15:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:28,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T14:15:28,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742072_1248 (size=12001) 2024-11-16T14:15:28,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T14:15:28,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:28,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:28,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:28,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766588796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766588796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766588796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:28,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:28,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:28,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T14:15:28,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:28,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:28,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:28,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:28,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T14:15:29,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/c5069cd0846846c8bd9307c8d1e15792 2024-11-16T14:15:29,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/5ed31532b1bf4632bd97afd095359a03 is 50, key is test_row_0/C:col10/1731766527496/Put/seqid=0 2024-11-16T14:15:29,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T14:15:29,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:29,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:29,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:29,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:29,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:29,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742073_1249 (size=12001) 2024-11-16T14:15:29,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/5ed31532b1bf4632bd97afd095359a03 2024-11-16T14:15:29,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/8b63648d02f046c595055cab0133fb1f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8b63648d02f046c595055cab0133fb1f 2024-11-16T14:15:29,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8b63648d02f046c595055cab0133fb1f, entries=200, sequenceid=68, filesize=14.0 K 2024-11-16T14:15:29,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/c5069cd0846846c8bd9307c8d1e15792 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c5069cd0846846c8bd9307c8d1e15792 2024-11-16T14:15:29,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c5069cd0846846c8bd9307c8d1e15792, entries=150, sequenceid=68, filesize=11.7 K 2024-11-16T14:15:29,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/5ed31532b1bf4632bd97afd095359a03 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/5ed31532b1bf4632bd97afd095359a03 2024-11-16T14:15:29,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/5ed31532b1bf4632bd97afd095359a03, entries=150, sequenceid=68, filesize=11.7 K 2024-11-16T14:15:29,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 5bda05ae08ffa563beaf32369a8bf2d2 in 983ms, sequenceid=68, compaction requested=false 2024-11-16T14:15:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:29,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T14:15:29,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:29,220 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:15:29,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:29,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:29,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:29,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:29,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:29,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:29,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f590652cfbe743329d0704b8cf3f2817 is 50, key is test_row_0/A:col10/1731766528159/Put/seqid=0 2024-11-16T14:15:29,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742074_1250 (size=12001) 2024-11-16T14:15:29,274 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f590652cfbe743329d0704b8cf3f2817 2024-11-16T14:15:29,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/0201ce8093d5402a8debd922af29aa87 is 50, key is test_row_0/B:col10/1731766528159/Put/seqid=0 2024-11-16T14:15:29,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742075_1251 (size=12001) 2024-11-16T14:15:29,299 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/0201ce8093d5402a8debd922af29aa87 2024-11-16T14:15:29,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/5bb2fb2880934d9c8011d771fd02d1bb is 50, key is test_row_0/C:col10/1731766528159/Put/seqid=0 2024-11-16T14:15:29,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:29,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742076_1252 (size=12001) 2024-11-16T14:15:29,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766589363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:29,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766589363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:29,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766589368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T14:15:29,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766589476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:29,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766589476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:29,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:29,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766589486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:29,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:29,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766589685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:29,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766589685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:29,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766589696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:29,714 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/5bb2fb2880934d9c8011d771fd02d1bb 2024-11-16T14:15:29,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f590652cfbe743329d0704b8cf3f2817 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f590652cfbe743329d0704b8cf3f2817 2024-11-16T14:15:29,722 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f590652cfbe743329d0704b8cf3f2817, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T14:15:29,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/0201ce8093d5402a8debd922af29aa87 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0201ce8093d5402a8debd922af29aa87 2024-11-16T14:15:29,726 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0201ce8093d5402a8debd922af29aa87, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T14:15:29,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/5bb2fb2880934d9c8011d771fd02d1bb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/5bb2fb2880934d9c8011d771fd02d1bb 2024-11-16T14:15:29,730 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/5bb2fb2880934d9c8011d771fd02d1bb, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T14:15:29,731 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5bda05ae08ffa563beaf32369a8bf2d2 in 511ms, sequenceid=82, compaction requested=true 2024-11-16T14:15:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-16T14:15:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-16T14:15:29,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-16T14:15:29,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4400 sec 2024-11-16T14:15:29,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4460 sec 2024-11-16T14:15:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:29,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=114.05 KB heapSize=340.13 KB 2024-11-16T14:15:30,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766529332, max timestamp=1731766529998Num uniques 859; 2024-11-16T14:15:30,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766529332, max timestamp=1731766529998Num uniques 859; 2024-11-16T14:15:30,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:30,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:30,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766529332, max timestamp=1731766529998Num uniques 859; 2024-11-16T14:15:30,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:30,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/8ef7173a7e674f8d9bc4f2f0105a71a8 is 50, key is test_row_0/A:col10/1731766529998/Put/seqid=0 2024-11-16T14:15:30,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742077_1253 (size=14341) 2024-11-16T14:15:30,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766530024, max timestamp=1731766530033Num uniques 859; 2024-11-16T14:15:30,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766530024, max timestamp=1731766530033Num uniques 859; 2024-11-16T14:15:30,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766530024, max timestamp=1731766530033Num uniques 859; 2024-11-16T14:15:30,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:30,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:30,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:30,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731766530083, max timestamp=1731766530102Num uniques 1068; 2024-11-16T14:15:30,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T14:15:30,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731766530083, max timestamp=1731766530102Num uniques 1068; 2024-11-16T14:15:30,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T14:15:30,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731766530083, max timestamp=1731766530102Num uniques 1068; 2024-11-16T14:15:30,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T14:15:30,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731766530119, max timestamp=1731766530140Num uniques 1027; 2024-11-16T14:15:30,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731766530119, max timestamp=1731766530140Num uniques 1027; 2024-11-16T14:15:30,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-16T14:15:30,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-16T14:15:30,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731766530119, max timestamp=1731766530140Num uniques 1027; 2024-11-16T14:15:30,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-16T14:15:30,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731766530156, max timestamp=1731766530176Num uniques 1086; 2024-11-16T14:15:30,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731766530156, max timestamp=1731766530176Num uniques 1086; 2024-11-16T14:15:30,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-16T14:15:30,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-16T14:15:30,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731766530156, max timestamp=1731766530176Num uniques 1086; 2024-11-16T14:15:30,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-16T14:15:30,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731766530202, max timestamp=1731766530212Num uniques 1095; 2024-11-16T14:15:30,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:30,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-16T14:15:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766590210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:30,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731766530202, max timestamp=1731766530212Num uniques 1095; 2024-11-16T14:15:30,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-16T14:15:30,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731766530202, max timestamp=1731766530212Num uniques 1095; 2024-11-16T14:15:30,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-16T14:15:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:30,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731766530250, max timestamp=1731766530270Num uniques 1104; 2024-11-16T14:15:30,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-16T14:15:30,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731766530250, max timestamp=1731766530270Num uniques 1104; 2024-11-16T14:15:30,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731766530250, max timestamp=1731766530270Num uniques 1104; 2024-11-16T14:15:30,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-16T14:15:30,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-16T14:15:30,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731766530309, max timestamp=1731766530331Num uniques 1013; 2024-11-16T14:15:30,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=12038, suffix heap size=133320,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=263 2024-11-16T14:15:30,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731766530309, max timestamp=1731766530331Num uniques 1063; 2024-11-16T14:15:30,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731766530309, max timestamp=1731766530331Num uniques 1013; 2024-11-16T14:15:30,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=12038, suffix heap size=138520,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=263 2024-11-16T14:15:30,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=12038, suffix heap size=133320,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=263 2024-11-16T14:15:30,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766590367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:30,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766590369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:30,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766590374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:30,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731766530365, max timestamp=1731766530372Num uniques 1122; 2024-11-16T14:15:30,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-16T14:15:30,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731766530365, max timestamp=1731766530372Num uniques 1122; 2024-11-16T14:15:30,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731766530365, max timestamp=1731766530372Num uniques 1122; 2024-11-16T14:15:30,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-16T14:15:30,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-16T14:15:30,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:30,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/8ef7173a7e674f8d9bc4f2f0105a71a8 2024-11-16T14:15:30,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/7083d56efb3f420aa687cea9988b4292 is 50, key is test_row_0/B:col10/1731766529998/Put/seqid=0 2024-11-16T14:15:30,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742078_1254 (size=12001) 2024-11-16T14:15:30,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/7083d56efb3f420aa687cea9988b4292 2024-11-16T14:15:30,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/43b5955156364fed96ed1db43ed60cdd is 50, key is test_row_0/C:col10/1731766529998/Put/seqid=0 2024-11-16T14:15:30,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742079_1255 (size=12001) 2024-11-16T14:15:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T14:15:30,437 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:30,437 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-16T14:15:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T14:15:30,439 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:30,439 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:30,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:30,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731766530485, max timestamp=1731766530515Num uniques 1031; 2024-11-16T14:15:30,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-16T14:15:30,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731766530485, max timestamp=1731766530515Num uniques 1031; 2024-11-16T14:15:30,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10573, totalHeapSize=24264, min timestamp=1731766530485, max timestamp=1731766530515Num uniques 1031; 2024-11-16T14:15:30,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-16T14:15:30,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47213, new segment data size=10573, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-16T14:15:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T14:15:30,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731766530536, max timestamp=1731766530550Num uniques 1090; 2024-11-16T14:15:30,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731766530536, max timestamp=1731766530550Num uniques 1090; 2024-11-16T14:15:30,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-16T14:15:30,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-16T14:15:30,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731766530536, max timestamp=1731766530550Num uniques 1090; 2024-11-16T14:15:30,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-16T14:15:30,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766590585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:30,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766590585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:30,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731766530575, max timestamp=1731766530590Num uniques 1149; 2024-11-16T14:15:30,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=9114, suffix heap size=147464,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=199 2024-11-16T14:15:30,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T14:15:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:30,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731766530575, max timestamp=1731766530590Num uniques 1149; 2024-11-16T14:15:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:30,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731766530575, max timestamp=1731766530590Num uniques 1149; 2024-11-16T14:15:30,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=9114, suffix heap size=147464,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=199 2024-11-16T14:15:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:30,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=9114, suffix heap size=147464,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=199 2024-11-16T14:15:30,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:30,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731766530665, max timestamp=1731766530702Num uniques 1058; 2024-11-16T14:15:30,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731766530665, max timestamp=1731766530702Num uniques 1058; 2024-11-16T14:15:30,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-16T14:15:30,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-16T14:15:30,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731766530665, max timestamp=1731766530702Num uniques 1058; 2024-11-16T14:15:30,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-16T14:15:30,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T14:15:30,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:30,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:30,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:30,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:30,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T14:15:30,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731766530727, max timestamp=1731766530772Num uniques 1017; 2024-11-16T14:15:30,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-16T14:15:30,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731766530727, max timestamp=1731766530772Num uniques 1017; 2024-11-16T14:15:30,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731766530727, max timestamp=1731766530772Num uniques 1017; 2024-11-16T14:15:30,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-16T14:15:30,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-16T14:15:30,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731766530804, max timestamp=1731766530814Num uniques 1076; 2024-11-16T14:15:30,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731766530804, max timestamp=1731766530814Num uniques 1076; 2024-11-16T14:15:30,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-16T14:15:30,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-16T14:15:30,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731766530804, max timestamp=1731766530814Num uniques 1076; 2024-11-16T14:15:30,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-16T14:15:30,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/43b5955156364fed96ed1db43ed60cdd 2024-11-16T14:15:30,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/8ef7173a7e674f8d9bc4f2f0105a71a8 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8ef7173a7e674f8d9bc4f2f0105a71a8 2024-11-16T14:15:30,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8ef7173a7e674f8d9bc4f2f0105a71a8, entries=200, sequenceid=103, filesize=14.0 K 2024-11-16T14:15:30,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/7083d56efb3f420aa687cea9988b4292 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/7083d56efb3f420aa687cea9988b4292 2024-11-16T14:15:30,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731766530839, max timestamp=1731766530852Num uniques 1085; 2024-11-16T14:15:30,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-11-16T14:15:30,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731766530839, max timestamp=1731766530852Num uniques 1085; 2024-11-16T14:15:30,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-11-16T14:15:30,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731766530839, max timestamp=1731766530852Num uniques 1085; 2024-11-16T14:15:30,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-11-16T14:15:30,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/7083d56efb3f420aa687cea9988b4292, entries=150, sequenceid=103, filesize=11.7 K 2024-11-16T14:15:30,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/43b5955156364fed96ed1db43ed60cdd as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/43b5955156364fed96ed1db43ed60cdd 2024-11-16T14:15:30,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/43b5955156364fed96ed1db43ed60cdd, entries=150, sequenceid=103, filesize=11.7 K 2024-11-16T14:15:30,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=53.67 KB/54960 for 5bda05ae08ffa563beaf32369a8bf2d2 in 876ms, sequenceid=103, compaction requested=true 2024-11-16T14:15:30,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:30,875 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:15:30,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:30,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:30,876 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:15:30,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:30,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:30,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:30,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:30,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=67.09 KB heapSize=183.33 KB 2024-11-16T14:15:30,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:30,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:30,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:30,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:30,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:30,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:30,878 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:15:30,878 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/A is initiating minor compaction (all files) 2024-11-16T14:15:30,878 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/A in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:30,878 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/2f42ef8fbd0d474695b9e6fc33ba2a1e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8b63648d02f046c595055cab0133fb1f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f590652cfbe743329d0704b8cf3f2817, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8ef7173a7e674f8d9bc4f2f0105a71a8] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=51.5 K 2024-11-16T14:15:30,878 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:15:30,878 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/B is initiating minor compaction (all files) 2024-11-16T14:15:30,878 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/B in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:30,879 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/3200bd77247947cbb43d1c99b5b3da8c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c5069cd0846846c8bd9307c8d1e15792, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0201ce8093d5402a8debd922af29aa87, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/7083d56efb3f420aa687cea9988b4292] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=47.0 K 2024-11-16T14:15:30,879 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3200bd77247947cbb43d1c99b5b3da8c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731766526322 2024-11-16T14:15:30,880 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f42ef8fbd0d474695b9e6fc33ba2a1e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731766526322 2024-11-16T14:15:30,880 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting c5069cd0846846c8bd9307c8d1e15792, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731766527496 2024-11-16T14:15:30,881 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0201ce8093d5402a8debd922af29aa87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731766528152 2024-11-16T14:15:30,881 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b63648d02f046c595055cab0133fb1f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731766527483 2024-11-16T14:15:30,881 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7083d56efb3f420aa687cea9988b4292, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731766529336 2024-11-16T14:15:30,882 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f590652cfbe743329d0704b8cf3f2817, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731766528152 2024-11-16T14:15:30,883 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ef7173a7e674f8d9bc4f2f0105a71a8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731766529332 2024-11-16T14:15:30,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/bd1a5c718f934360969ea94163000e24 is 50, key is test_row_0/A:col10/1731766530871/Put/seqid=0 2024-11-16T14:15:30,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T14:15:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:30,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:30,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742080_1256 (size=14741) 2024-11-16T14:15:30,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/bd1a5c718f934360969ea94163000e24 2024-11-16T14:15:30,912 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:30,913 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/cb0811d09fe045df9759179ba82f621c is 50, key is test_row_0/B:col10/1731766529998/Put/seqid=0 2024-11-16T14:15:30,923 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#A#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:30,924 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/214101f1f2544109bd8ff747902a4cf2 is 50, key is test_row_0/A:col10/1731766529998/Put/seqid=0 2024-11-16T14:15:30,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/66135e836e404f36b1526ba744254341 is 50, key is test_row_0/B:col10/1731766530871/Put/seqid=0 2024-11-16T14:15:30,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742081_1257 (size=12241) 2024-11-16T14:15:30,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742082_1258 (size=12241) 2024-11-16T14:15:30,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742083_1259 (size=12301) 2024-11-16T14:15:30,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766590941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:30,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766590950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766590952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T14:15:31,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:31,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:31,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:31,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:31,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:31,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766591055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T14:15:31,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766591066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766591066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T14:15:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:31,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:31,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766591265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766591275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766591275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,332 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/cb0811d09fe045df9759179ba82f621c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/cb0811d09fe045df9759179ba82f621c 2024-11-16T14:15:31,337 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/B of 5bda05ae08ffa563beaf32369a8bf2d2 into cb0811d09fe045df9759179ba82f621c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:31,337 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:31,337 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/B, priority=12, startTime=1731766530876; duration=0sec 2024-11-16T14:15:31,337 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:31,337 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:B 2024-11-16T14:15:31,337 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:15:31,338 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:15:31,338 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/C is initiating minor compaction (all files) 2024-11-16T14:15:31,338 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/C in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:31,338 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/acfdba2062944cef8bb8cbec0adf48d3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/5ed31532b1bf4632bd97afd095359a03, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/5bb2fb2880934d9c8011d771fd02d1bb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/43b5955156364fed96ed1db43ed60cdd] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=47.0 K 2024-11-16T14:15:31,339 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting acfdba2062944cef8bb8cbec0adf48d3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731766526322 2024-11-16T14:15:31,339 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed31532b1bf4632bd97afd095359a03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731766527496 2024-11-16T14:15:31,341 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bb2fb2880934d9c8011d771fd02d1bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731766528152 2024-11-16T14:15:31,341 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 43b5955156364fed96ed1db43ed60cdd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731766529336 2024-11-16T14:15:31,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/66135e836e404f36b1526ba744254341 2024-11-16T14:15:31,347 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/214101f1f2544109bd8ff747902a4cf2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/214101f1f2544109bd8ff747902a4cf2 2024-11-16T14:15:31,349 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#C#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:31,350 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/9cc3df17ac014202870c4d835d5f1c56 is 50, key is test_row_0/C:col10/1731766529998/Put/seqid=0 2024-11-16T14:15:31,354 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/A of 5bda05ae08ffa563beaf32369a8bf2d2 into 214101f1f2544109bd8ff747902a4cf2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:31,354 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:31,354 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/A, priority=12, startTime=1731766530875; duration=0sec 2024-11-16T14:15:31,354 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:31,354 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:A 2024-11-16T14:15:31,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/f4fc456462c34dd7ace38268c7f222f3 is 50, key is test_row_0/C:col10/1731766530871/Put/seqid=0 2024-11-16T14:15:31,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T14:15:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:31,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:31,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742084_1260 (size=12241) 2024-11-16T14:15:31,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742085_1261 (size=12301) 2024-11-16T14:15:31,381 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/9cc3df17ac014202870c4d835d5f1c56 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/9cc3df17ac014202870c4d835d5f1c56 2024-11-16T14:15:31,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/f4fc456462c34dd7ace38268c7f222f3 2024-11-16T14:15:31,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/bd1a5c718f934360969ea94163000e24 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/bd1a5c718f934360969ea94163000e24 2024-11-16T14:15:31,389 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/C of 5bda05ae08ffa563beaf32369a8bf2d2 into 9cc3df17ac014202870c4d835d5f1c56(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:31,389 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:31,389 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/C, priority=12, startTime=1731766530876; duration=0sec 2024-11-16T14:15:31,390 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:31,390 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:C 2024-11-16T14:15:31,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/bd1a5c718f934360969ea94163000e24, entries=200, sequenceid=370, filesize=14.4 K 2024-11-16T14:15:31,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/66135e836e404f36b1526ba744254341 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/66135e836e404f36b1526ba744254341 2024-11-16T14:15:31,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/66135e836e404f36b1526ba744254341, entries=150, sequenceid=370, filesize=12.0 K 2024-11-16T14:15:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/f4fc456462c34dd7ace38268c7f222f3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/f4fc456462c34dd7ace38268c7f222f3 2024-11-16T14:15:31,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/f4fc456462c34dd7ace38268c7f222f3, entries=150, sequenceid=370, filesize=12.0 K 2024-11-16T14:15:31,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~183.28 KB/187680, currentSize=114.05 KB/116790 for 5bda05ae08ffa563beaf32369a8bf2d2 in 534ms, sequenceid=370, compaction requested=false 2024-11-16T14:15:31,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:31,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T14:15:31,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:31,516 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:15:31,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:31,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:31,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:31,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:31,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:31,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:31,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/19f4e192e86f406ea2790f7f861a78c2 is 50, key is test_row_0/A:col10/1731766530948/Put/seqid=0 2024-11-16T14:15:31,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742086_1262 (size=12301) 2024-11-16T14:15:31,532 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/19f4e192e86f406ea2790f7f861a78c2 2024-11-16T14:15:31,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/3708445ae27741b5afd99e557fdf6250 is 50, key is test_row_0/B:col10/1731766530948/Put/seqid=0 2024-11-16T14:15:31,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742087_1263 (size=12301) 2024-11-16T14:15:31,567 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/3708445ae27741b5afd99e557fdf6250 2024-11-16T14:15:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T14:15:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:31,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:31,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/4ca8e3179e6e4decb1a6a7640c268ead is 50, key is test_row_0/C:col10/1731766530948/Put/seqid=0 2024-11-16T14:15:31,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742088_1264 (size=12301) 2024-11-16T14:15:31,593 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/4ca8e3179e6e4decb1a6a7640c268ead 2024-11-16T14:15:31,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/19f4e192e86f406ea2790f7f861a78c2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/19f4e192e86f406ea2790f7f861a78c2 2024-11-16T14:15:31,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766591598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,611 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/19f4e192e86f406ea2790f7f861a78c2, entries=150, sequenceid=393, filesize=12.0 K 2024-11-16T14:15:31,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/3708445ae27741b5afd99e557fdf6250 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/3708445ae27741b5afd99e557fdf6250 2024-11-16T14:15:31,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766591610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766591610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,619 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/3708445ae27741b5afd99e557fdf6250, entries=150, sequenceid=393, filesize=12.0 K 2024-11-16T14:15:31,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/4ca8e3179e6e4decb1a6a7640c268ead as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/4ca8e3179e6e4decb1a6a7640c268ead 2024-11-16T14:15:31,630 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/4ca8e3179e6e4decb1a6a7640c268ead, entries=150, sequenceid=393, filesize=12.0 K 2024-11-16T14:15:31,631 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 5bda05ae08ffa563beaf32369a8bf2d2 in 115ms, sequenceid=393, compaction requested=true 2024-11-16T14:15:31,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:31,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:31,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-16T14:15:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-16T14:15:31,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-16T14:15:31,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1940 sec 2024-11-16T14:15:31,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1980 sec 2024-11-16T14:15:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:31,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T14:15:31,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:31,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:31,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:31,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:31,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:31,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:31,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/91efdc9e005f41fd94bf66702dedc2f9 is 50, key is test_row_0/A:col10/1731766531594/Put/seqid=0 2024-11-16T14:15:31,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742089_1265 (size=14741) 2024-11-16T14:15:31,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766591756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766591757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766591758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766591865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766591865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:31,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:31,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766591865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766592075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766592076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766592076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/91efdc9e005f41fd94bf66702dedc2f9 2024-11-16T14:15:32,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/02f79751ba85471681a051ed39d9c05e is 50, key is test_row_0/B:col10/1731766531594/Put/seqid=0 2024-11-16T14:15:32,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742090_1266 (size=12301) 2024-11-16T14:15:32,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/02f79751ba85471681a051ed39d9c05e 2024-11-16T14:15:32,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/71fb32536491404886d81d2c984f8673 is 50, key is test_row_0/C:col10/1731766531594/Put/seqid=0 2024-11-16T14:15:32,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742091_1267 (size=12301) 2024-11-16T14:15:32,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/71fb32536491404886d81d2c984f8673 2024-11-16T14:15:32,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/91efdc9e005f41fd94bf66702dedc2f9 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/91efdc9e005f41fd94bf66702dedc2f9 2024-11-16T14:15:32,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/91efdc9e005f41fd94bf66702dedc2f9, entries=200, sequenceid=407, filesize=14.4 K 2024-11-16T14:15:32,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/02f79751ba85471681a051ed39d9c05e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/02f79751ba85471681a051ed39d9c05e 2024-11-16T14:15:32,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/02f79751ba85471681a051ed39d9c05e, entries=150, sequenceid=407, filesize=12.0 K 2024-11-16T14:15:32,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/71fb32536491404886d81d2c984f8673 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/71fb32536491404886d81d2c984f8673 2024-11-16T14:15:32,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/71fb32536491404886d81d2c984f8673, entries=150, sequenceid=407, filesize=12.0 K 2024-11-16T14:15:32,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5bda05ae08ffa563beaf32369a8bf2d2 in 533ms, sequenceid=407, compaction requested=true 2024-11-16T14:15:32,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:32,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:32,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:32,250 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:15:32,250 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:15:32,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:32,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:32,252 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:15:32,252 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/B is initiating minor compaction (all files) 2024-11-16T14:15:32,252 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/B in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:32,252 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/cb0811d09fe045df9759179ba82f621c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/66135e836e404f36b1526ba744254341, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/3708445ae27741b5afd99e557fdf6250, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/02f79751ba85471681a051ed39d9c05e] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=48.0 K 2024-11-16T14:15:32,252 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting cb0811d09fe045df9759179ba82f621c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731766529336 2024-11-16T14:15:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:32,254 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 66135e836e404f36b1526ba744254341, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731766530865 2024-11-16T14:15:32,255 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:15:32,255 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3708445ae27741b5afd99e557fdf6250, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731766530938 2024-11-16T14:15:32,255 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/A is initiating minor compaction (all files) 2024-11-16T14:15:32,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:32,255 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/A in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:32,255 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/214101f1f2544109bd8ff747902a4cf2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/bd1a5c718f934360969ea94163000e24, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/19f4e192e86f406ea2790f7f861a78c2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/91efdc9e005f41fd94bf66702dedc2f9] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=52.8 K 2024-11-16T14:15:32,255 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f79751ba85471681a051ed39d9c05e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731766531594 2024-11-16T14:15:32,255 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 214101f1f2544109bd8ff747902a4cf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731766529336 2024-11-16T14:15:32,256 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd1a5c718f934360969ea94163000e24, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731766530865 2024-11-16T14:15:32,256 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19f4e192e86f406ea2790f7f861a78c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731766530938 2024-11-16T14:15:32,256 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91efdc9e005f41fd94bf66702dedc2f9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731766531594 2024-11-16T14:15:32,272 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:32,273 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/4bfbfdd0439946048fdb8d34e53b4c73 is 50, key is test_row_0/B:col10/1731766531594/Put/seqid=0 2024-11-16T14:15:32,288 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#A#compaction#226 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:32,288 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f7c216129aa4417fad9442dd029d757b is 50, key is test_row_0/A:col10/1731766531594/Put/seqid=0 2024-11-16T14:15:32,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742092_1268 (size=12677) 2024-11-16T14:15:32,357 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/4bfbfdd0439946048fdb8d34e53b4c73 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/4bfbfdd0439946048fdb8d34e53b4c73 2024-11-16T14:15:32,365 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/B of 5bda05ae08ffa563beaf32369a8bf2d2 into 4bfbfdd0439946048fdb8d34e53b4c73(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:32,365 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:32,365 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/B, priority=12, startTime=1731766532250; duration=0sec 2024-11-16T14:15:32,365 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:32,365 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:B 2024-11-16T14:15:32,365 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:15:32,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742093_1269 (size=12677) 2024-11-16T14:15:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:32,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:15:32,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:32,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:32,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:32,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:32,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:32,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:32,375 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:15:32,375 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/C is initiating minor compaction (all files) 2024-11-16T14:15:32,375 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/C in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:32,376 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/9cc3df17ac014202870c4d835d5f1c56, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/f4fc456462c34dd7ace38268c7f222f3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/4ca8e3179e6e4decb1a6a7640c268ead, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/71fb32536491404886d81d2c984f8673] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=48.0 K 2024-11-16T14:15:32,376 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cc3df17ac014202870c4d835d5f1c56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731766529336 2024-11-16T14:15:32,377 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f4fc456462c34dd7ace38268c7f222f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731766530865 2024-11-16T14:15:32,377 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ca8e3179e6e4decb1a6a7640c268ead, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731766530938 2024-11-16T14:15:32,377 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 71fb32536491404886d81d2c984f8673, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731766531594 2024-11-16T14:15:32,379 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f7c216129aa4417fad9442dd029d757b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f7c216129aa4417fad9442dd029d757b 2024-11-16T14:15:32,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/1221779a3ef54311a6b754b0454bf6c6 is 50, key is test_row_0/A:col10/1731766532366/Put/seqid=0 2024-11-16T14:15:32,393 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/A of 5bda05ae08ffa563beaf32369a8bf2d2 into f7c216129aa4417fad9442dd029d757b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:32,393 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:32,394 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/A, priority=12, startTime=1731766532250; duration=0sec 2024-11-16T14:15:32,394 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:32,394 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:A 2024-11-16T14:15:32,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766592407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,418 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#C#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:32,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766592410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,419 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/683cbe533b7a4e12a8c83e38a19b43c2 is 50, key is test_row_0/C:col10/1731766531594/Put/seqid=0 2024-11-16T14:15:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766592412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766592412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766592413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742094_1270 (size=14741) 2024-11-16T14:15:32,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742095_1271 (size=12677) 2024-11-16T14:15:32,459 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/683cbe533b7a4e12a8c83e38a19b43c2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/683cbe533b7a4e12a8c83e38a19b43c2 2024-11-16T14:15:32,466 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/C of 5bda05ae08ffa563beaf32369a8bf2d2 into 683cbe533b7a4e12a8c83e38a19b43c2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:32,466 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:32,466 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/C, priority=12, startTime=1731766532251; duration=0sec 2024-11-16T14:15:32,466 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:32,466 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:C 2024-11-16T14:15:32,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766592516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766592525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766592525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766592526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766592526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T14:15:32,586 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:32,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-16T14:15:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T14:15:32,589 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:32,590 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:32,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T14:15:32,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766592726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766592736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766592736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766592736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766592736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:32,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:32,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T14:15:32,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:32,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:32,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:32,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:32,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:32,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/1221779a3ef54311a6b754b0454bf6c6 2024-11-16T14:15:32,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/98bfd28af1cd4991b0f2711cc5b0d01d is 50, key is test_row_0/B:col10/1731766532366/Put/seqid=0 2024-11-16T14:15:32,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742096_1272 (size=12301) 2024-11-16T14:15:32,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/98bfd28af1cd4991b0f2711cc5b0d01d 2024-11-16T14:15:32,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/0a02b03cdfbf4fd2851ab8ddc28e0c03 is 50, key is test_row_0/C:col10/1731766532366/Put/seqid=0 2024-11-16T14:15:32,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T14:15:32,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:32,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:32,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:32,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:32,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T14:15:32,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742097_1273 (size=12301) 2024-11-16T14:15:32,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/0a02b03cdfbf4fd2851ab8ddc28e0c03 2024-11-16T14:15:32,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/1221779a3ef54311a6b754b0454bf6c6 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/1221779a3ef54311a6b754b0454bf6c6 2024-11-16T14:15:32,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/1221779a3ef54311a6b754b0454bf6c6, entries=200, sequenceid=427, filesize=14.4 K 2024-11-16T14:15:32,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/98bfd28af1cd4991b0f2711cc5b0d01d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/98bfd28af1cd4991b0f2711cc5b0d01d 2024-11-16T14:15:32,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/98bfd28af1cd4991b0f2711cc5b0d01d, entries=150, sequenceid=427, filesize=12.0 K 2024-11-16T14:15:32,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/0a02b03cdfbf4fd2851ab8ddc28e0c03 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/0a02b03cdfbf4fd2851ab8ddc28e0c03 2024-11-16T14:15:32,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/0a02b03cdfbf4fd2851ab8ddc28e0c03, entries=150, sequenceid=427, filesize=12.0 K 2024-11-16T14:15:32,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=80.51 KB/82440 for 5bda05ae08ffa563beaf32369a8bf2d2 in 580ms, sequenceid=427, compaction requested=false 2024-11-16T14:15:32,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:33,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T14:15:33,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:33,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:33,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:33,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:33,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:33,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:33,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/ec9fa979677a46c7971394d346462833 is 50, key is test_row_0/A:col10/1731766532379/Put/seqid=0 2024-11-16T14:15:33,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T14:15:33,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:33,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742098_1274 (size=14741) 2024-11-16T14:15:33,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766593065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766593067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766593073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766593074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766593085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766593175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766593176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766593196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766593196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766593205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T14:15:33,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:33,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T14:15:33,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T14:15:33,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:33,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766593385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766593385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766593406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766593406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766593416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/ec9fa979677a46c7971394d346462833 2024-11-16T14:15:33,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/09d3cedc940444b49bb241782136f0c2 is 50, key is test_row_0/B:col10/1731766532379/Put/seqid=0 2024-11-16T14:15:33,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T14:15:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742099_1275 (size=12301) 2024-11-16T14:15:33,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/09d3cedc940444b49bb241782136f0c2 2024-11-16T14:15:33,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/84cb95533db64f1ea8dd13fbf73a53e9 is 50, key is test_row_0/C:col10/1731766532379/Put/seqid=0 2024-11-16T14:15:33,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742100_1276 (size=12301) 2024-11-16T14:15:33,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T14:15:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766593696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766593696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766593715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766593715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T14:15:33,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766593726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:33,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:33,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T14:15:33,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:33,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:33,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/84cb95533db64f1ea8dd13fbf73a53e9 2024-11-16T14:15:33,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/ec9fa979677a46c7971394d346462833 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/ec9fa979677a46c7971394d346462833 2024-11-16T14:15:33,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/ec9fa979677a46c7971394d346462833, entries=200, sequenceid=445, filesize=14.4 K 2024-11-16T14:15:33,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/09d3cedc940444b49bb241782136f0c2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/09d3cedc940444b49bb241782136f0c2 2024-11-16T14:15:33,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/09d3cedc940444b49bb241782136f0c2, entries=150, sequenceid=445, filesize=12.0 K 2024-11-16T14:15:33,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/84cb95533db64f1ea8dd13fbf73a53e9 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/84cb95533db64f1ea8dd13fbf73a53e9 2024-11-16T14:15:33,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/84cb95533db64f1ea8dd13fbf73a53e9, entries=150, sequenceid=445, filesize=12.0 K 2024-11-16T14:15:33,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 5bda05ae08ffa563beaf32369a8bf2d2 in 928ms, sequenceid=445, compaction requested=true 2024-11-16T14:15:33,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:33,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:33,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:33,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:33,967 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:33,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:33,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:33,967 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:33,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:33,968 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42159 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:33,968 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:33,968 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/B is initiating minor compaction (all files) 2024-11-16T14:15:33,968 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/A is initiating minor compaction (all files) 2024-11-16T14:15:33,968 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/B in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,968 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/A in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,968 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/4bfbfdd0439946048fdb8d34e53b4c73, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/98bfd28af1cd4991b0f2711cc5b0d01d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/09d3cedc940444b49bb241782136f0c2] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.4 K 2024-11-16T14:15:33,968 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f7c216129aa4417fad9442dd029d757b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/1221779a3ef54311a6b754b0454bf6c6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/ec9fa979677a46c7971394d346462833] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=41.2 K 2024-11-16T14:15:33,969 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bfbfdd0439946048fdb8d34e53b4c73, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731766531594 2024-11-16T14:15:33,969 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7c216129aa4417fad9442dd029d757b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731766531594 2024-11-16T14:15:33,969 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1221779a3ef54311a6b754b0454bf6c6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731766531754 2024-11-16T14:15:33,969 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 98bfd28af1cd4991b0f2711cc5b0d01d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731766531754 2024-11-16T14:15:33,969 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 09d3cedc940444b49bb241782136f0c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731766532379 2024-11-16T14:15:33,969 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec9fa979677a46c7971394d346462833, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731766532379 2024-11-16T14:15:33,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T14:15:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:33,976 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T14:15:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:33,979 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:33,979 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#B#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:33,979 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/564265cd975c4028b214ce5fd44a90eb is 50, key is test_row_0/A:col10/1731766532379/Put/seqid=0 2024-11-16T14:15:33,980 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/822a232bfada435aae2536c95b07f6f3 is 50, key is test_row_0/B:col10/1731766532379/Put/seqid=0 2024-11-16T14:15:34,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/c493a1e120e6467dbe17058aad6cff46 is 50, key is test_row_0/A:col10/1731766533071/Put/seqid=0 2024-11-16T14:15:34,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742102_1278 (size=12779) 2024-11-16T14:15:34,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742103_1279 (size=12301) 2024-11-16T14:15:34,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742101_1277 (size=12779) 2024-11-16T14:15:34,034 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/564265cd975c4028b214ce5fd44a90eb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/564265cd975c4028b214ce5fd44a90eb 2024-11-16T14:15:34,042 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/c493a1e120e6467dbe17058aad6cff46 2024-11-16T14:15:34,045 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/A of 5bda05ae08ffa563beaf32369a8bf2d2 into 564265cd975c4028b214ce5fd44a90eb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:34,045 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:34,045 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/A, priority=13, startTime=1731766533967; duration=0sec 2024-11-16T14:15:34,046 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:34,046 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:A 2024-11-16T14:15:34,046 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:34,049 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:34,049 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/C is initiating minor compaction (all files) 2024-11-16T14:15:34,049 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/C in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:34,049 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/683cbe533b7a4e12a8c83e38a19b43c2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/0a02b03cdfbf4fd2851ab8ddc28e0c03, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/84cb95533db64f1ea8dd13fbf73a53e9] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.4 K 2024-11-16T14:15:34,051 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 683cbe533b7a4e12a8c83e38a19b43c2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731766531594 2024-11-16T14:15:34,051 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a02b03cdfbf4fd2851ab8ddc28e0c03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731766531754 2024-11-16T14:15:34,052 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84cb95533db64f1ea8dd13fbf73a53e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731766532379 2024-11-16T14:15:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/a34e7073f70040a4b439e9f6479c5a7f is 50, key is test_row_0/B:col10/1731766533071/Put/seqid=0 2024-11-16T14:15:34,075 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#C#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:34,079 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/1ac16c2a8b9546fbad3244426bf1f984 is 50, key is test_row_0/C:col10/1731766532379/Put/seqid=0 2024-11-16T14:15:34,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742105_1281 (size=12779) 2024-11-16T14:15:34,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742104_1280 (size=12301) 2024-11-16T14:15:34,122 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/a34e7073f70040a4b439e9f6479c5a7f 2024-11-16T14:15:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/523b653880684c16ada131e290cba11b is 50, key is test_row_0/C:col10/1731766533071/Put/seqid=0 2024-11-16T14:15:34,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742106_1282 (size=12301) 2024-11-16T14:15:34,175 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/523b653880684c16ada131e290cba11b 2024-11-16T14:15:34,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/c493a1e120e6467dbe17058aad6cff46 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/c493a1e120e6467dbe17058aad6cff46 2024-11-16T14:15:34,191 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/c493a1e120e6467dbe17058aad6cff46, entries=150, sequenceid=462, filesize=12.0 K 2024-11-16T14:15:34,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/a34e7073f70040a4b439e9f6479c5a7f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/a34e7073f70040a4b439e9f6479c5a7f 2024-11-16T14:15:34,203 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/a34e7073f70040a4b439e9f6479c5a7f, entries=150, sequenceid=462, filesize=12.0 K 2024-11-16T14:15:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/523b653880684c16ada131e290cba11b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/523b653880684c16ada131e290cba11b 2024-11-16T14:15:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:34,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:34,211 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/523b653880684c16ada131e290cba11b, entries=150, sequenceid=462, filesize=12.0 K 2024-11-16T14:15:34,212 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=6.71 KB/6870 for 5bda05ae08ffa563beaf32369a8bf2d2 in 236ms, sequenceid=462, compaction requested=false 2024-11-16T14:15:34,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:34,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:34,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-16T14:15:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-16T14:15:34,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-16T14:15:34,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6240 sec 2024-11-16T14:15:34,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6330 sec 2024-11-16T14:15:34,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:34,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:15:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:34,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:34,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/c22d3659d1b2476087c1eb22cf599920 is 50, key is test_row_0/A:col10/1731766534286/Put/seqid=0 2024-11-16T14:15:34,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742107_1283 (size=14741) 2024-11-16T14:15:34,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766594312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766594314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766594314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766594315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766594322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766594425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766594425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766594425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,431 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/822a232bfada435aae2536c95b07f6f3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/822a232bfada435aae2536c95b07f6f3 2024-11-16T14:15:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766594425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,435 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/B of 5bda05ae08ffa563beaf32369a8bf2d2 into 822a232bfada435aae2536c95b07f6f3(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:34,435 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:34,435 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/B, priority=13, startTime=1731766533967; duration=0sec 2024-11-16T14:15:34,435 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:34,435 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:B 2024-11-16T14:15:34,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766594436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,509 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/1ac16c2a8b9546fbad3244426bf1f984 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/1ac16c2a8b9546fbad3244426bf1f984 2024-11-16T14:15:34,513 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/C of 5bda05ae08ffa563beaf32369a8bf2d2 into 1ac16c2a8b9546fbad3244426bf1f984(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:34,513 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:34,513 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/C, priority=13, startTime=1731766533967; duration=0sec 2024-11-16T14:15:34,513 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:34,513 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:C 2024-11-16T14:15:34,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766594636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766594636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766594636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766594636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766594646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/c22d3659d1b2476087c1eb22cf599920 2024-11-16T14:15:34,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/18fe1a4652874193a9c323331ce92627 is 50, key is test_row_0/B:col10/1731766534286/Put/seqid=0 2024-11-16T14:15:34,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742108_1284 (size=12301) 2024-11-16T14:15:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T14:15:34,736 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:34,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-16T14:15:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T14:15:34,737 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:34,738 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:34,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T14:15:34,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T14:15:34,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:34,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:34,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:34,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:34,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:34,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766594945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766594945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766594945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766594945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:34,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766594956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:35,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T14:15:35,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:35,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:35,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:35,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:35,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T14:15:35,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/18fe1a4652874193a9c323331ce92627 2024-11-16T14:15:35,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/2a7b2b579daf4f27a5360934f797122d is 50, key is test_row_0/C:col10/1731766534286/Put/seqid=0 2024-11-16T14:15:35,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742109_1285 (size=12301) 2024-11-16T14:15:35,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T14:15:35,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:35,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:35,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:35,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:35,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:35,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T14:15:35,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:35,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:35,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:35,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:35,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T14:15:35,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766595456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:35,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766595456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:35,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766595456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:35,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766595456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:35,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766595480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:35,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T14:15:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:35,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:35,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:35,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/2a7b2b579daf4f27a5360934f797122d 2024-11-16T14:15:35,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/c22d3659d1b2476087c1eb22cf599920 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/c22d3659d1b2476087c1eb22cf599920 2024-11-16T14:15:35,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/c22d3659d1b2476087c1eb22cf599920, entries=200, sequenceid=476, filesize=14.4 K 2024-11-16T14:15:35,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/18fe1a4652874193a9c323331ce92627 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/18fe1a4652874193a9c323331ce92627 2024-11-16T14:15:35,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/18fe1a4652874193a9c323331ce92627, entries=150, sequenceid=476, filesize=12.0 K 2024-11-16T14:15:35,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/2a7b2b579daf4f27a5360934f797122d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/2a7b2b579daf4f27a5360934f797122d 2024-11-16T14:15:35,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/2a7b2b579daf4f27a5360934f797122d, entries=150, sequenceid=476, filesize=12.0 K 2024-11-16T14:15:35,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5bda05ae08ffa563beaf32369a8bf2d2 in 1278ms, sequenceid=476, compaction requested=true 2024-11-16T14:15:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:35,564 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:35,565 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:35,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:35,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:35,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:35,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:35,568 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:35,568 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:35,568 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/A is initiating minor compaction (all files) 2024-11-16T14:15:35,568 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/B is initiating minor compaction (all files) 2024-11-16T14:15:35,568 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/A in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:35,568 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/B in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:35,569 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/564265cd975c4028b214ce5fd44a90eb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/c493a1e120e6467dbe17058aad6cff46, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/c22d3659d1b2476087c1eb22cf599920] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=38.9 K 2024-11-16T14:15:35,569 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/822a232bfada435aae2536c95b07f6f3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/a34e7073f70040a4b439e9f6479c5a7f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/18fe1a4652874193a9c323331ce92627] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.5 K 2024-11-16T14:15:35,569 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 822a232bfada435aae2536c95b07f6f3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731766532379 2024-11-16T14:15:35,569 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 564265cd975c4028b214ce5fd44a90eb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731766532379 2024-11-16T14:15:35,569 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting a34e7073f70040a4b439e9f6479c5a7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731766533052 2024-11-16T14:15:35,569 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c493a1e120e6467dbe17058aad6cff46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731766533052 2024-11-16T14:15:35,570 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 18fe1a4652874193a9c323331ce92627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1731766534216 2024-11-16T14:15:35,570 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c22d3659d1b2476087c1eb22cf599920, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1731766534216 2024-11-16T14:15:35,578 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:35,579 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/ec00d167a93041bcbc4970d4051553c2 is 50, key is test_row_0/B:col10/1731766534286/Put/seqid=0 2024-11-16T14:15:35,591 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:35,591 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/3644d418fb4d46d38f5743caa17f04d4 is 50, key is test_row_0/A:col10/1731766534286/Put/seqid=0 2024-11-16T14:15:35,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742110_1286 (size=12881) 2024-11-16T14:15:35,636 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/ec00d167a93041bcbc4970d4051553c2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/ec00d167a93041bcbc4970d4051553c2 2024-11-16T14:15:35,640 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/B of 5bda05ae08ffa563beaf32369a8bf2d2 into ec00d167a93041bcbc4970d4051553c2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:35,641 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:35,641 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/B, priority=13, startTime=1731766535564; duration=0sec 2024-11-16T14:15:35,641 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:35,641 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:B 2024-11-16T14:15:35,641 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:35,642 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:35,642 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/C is initiating minor compaction (all files) 2024-11-16T14:15:35,642 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/C in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:35,642 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/1ac16c2a8b9546fbad3244426bf1f984, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/523b653880684c16ada131e290cba11b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/2a7b2b579daf4f27a5360934f797122d] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.5 K 2024-11-16T14:15:35,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742111_1287 (size=12881) 2024-11-16T14:15:35,644 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ac16c2a8b9546fbad3244426bf1f984, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731766532379 2024-11-16T14:15:35,645 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 523b653880684c16ada131e290cba11b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731766533052 2024-11-16T14:15:35,646 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a7b2b579daf4f27a5360934f797122d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1731766534216 2024-11-16T14:15:35,656 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/3644d418fb4d46d38f5743caa17f04d4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/3644d418fb4d46d38f5743caa17f04d4 2024-11-16T14:15:35,660 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#C#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:35,660 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/17823036a7154c208898608b37fe08bd is 50, key is test_row_0/C:col10/1731766534286/Put/seqid=0 2024-11-16T14:15:35,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T14:15:35,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:35,663 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:15:35,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:35,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:35,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:35,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:35,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:35,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:35,667 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/A of 5bda05ae08ffa563beaf32369a8bf2d2 into 3644d418fb4d46d38f5743caa17f04d4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:35,667 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:35,667 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/A, priority=13, startTime=1731766535564; duration=0sec 2024-11-16T14:15:35,667 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:35,667 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:A 2024-11-16T14:15:35,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f97d0ba766be4d0db873e649e60490b1 is 50, key is test_row_0/A:col10/1731766534311/Put/seqid=0 2024-11-16T14:15:35,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742112_1288 (size=12881) 2024-11-16T14:15:35,685 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/17823036a7154c208898608b37fe08bd as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/17823036a7154c208898608b37fe08bd 2024-11-16T14:15:35,690 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/C of 5bda05ae08ffa563beaf32369a8bf2d2 into 17823036a7154c208898608b37fe08bd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:35,690 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:35,690 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/C, priority=13, startTime=1731766535565; duration=0sec 2024-11-16T14:15:35,690 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:35,690 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:C 2024-11-16T14:15:35,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742113_1289 (size=12301) 2024-11-16T14:15:35,692 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f97d0ba766be4d0db873e649e60490b1 2024-11-16T14:15:35,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/192145a293414edca02adcec58eb1cf2 is 50, key is test_row_0/B:col10/1731766534311/Put/seqid=0 2024-11-16T14:15:35,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742114_1290 (size=12301) 2024-11-16T14:15:35,714 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/192145a293414edca02adcec58eb1cf2 2024-11-16T14:15:35,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/6296c15d265442a7a4840db17181af4e is 50, key is test_row_0/C:col10/1731766534311/Put/seqid=0 2024-11-16T14:15:35,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742115_1291 (size=12301) 2024-11-16T14:15:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T14:15:36,158 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/6296c15d265442a7a4840db17181af4e 2024-11-16T14:15:36,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f97d0ba766be4d0db873e649e60490b1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f97d0ba766be4d0db873e649e60490b1 2024-11-16T14:15:36,166 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f97d0ba766be4d0db873e649e60490b1, entries=150, sequenceid=498, filesize=12.0 K 2024-11-16T14:15:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/192145a293414edca02adcec58eb1cf2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/192145a293414edca02adcec58eb1cf2 2024-11-16T14:15:36,171 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/192145a293414edca02adcec58eb1cf2, entries=150, sequenceid=498, filesize=12.0 K 2024-11-16T14:15:36,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/6296c15d265442a7a4840db17181af4e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/6296c15d265442a7a4840db17181af4e 2024-11-16T14:15:36,183 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/6296c15d265442a7a4840db17181af4e, entries=150, sequenceid=498, filesize=12.0 K 2024-11-16T14:15:36,184 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 5bda05ae08ffa563beaf32369a8bf2d2 in 521ms, sequenceid=498, compaction requested=false 2024-11-16T14:15:36,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:36,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:36,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-16T14:15:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-16T14:15:36,189 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-16T14:15:36,189 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4490 sec 2024-11-16T14:15:36,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4540 sec 2024-11-16T14:15:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:36,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:15:36,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:36,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:36,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:36,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:36,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:36,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:36,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/4562844198e245c881e0777fbacf3eb1 is 50, key is test_row_0/A:col10/1731766536472/Put/seqid=0 2024-11-16T14:15:36,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742116_1292 (size=17181) 2024-11-16T14:15:36,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766536495, max timestamp=1731766536499Num uniques 859; 2024-11-16T14:15:36,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:36,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766536495, max timestamp=1731766536499Num uniques 859; 2024-11-16T14:15:36,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:36,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766536495, max timestamp=1731766536499Num uniques 859; 2024-11-16T14:15:36,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:36,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766596544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766596544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766596544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766596545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766596545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766596655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766596655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766596656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766596656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766596656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,720 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:15:36,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766596865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766596866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766596866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766596866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766596866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:36,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:36,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:36,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/4562844198e245c881e0777fbacf3eb1 2024-11-16T14:15:36,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/f18fe92a410e434c88d69d6fb2d2948f is 50, key is test_row_0/B:col10/1731766536472/Put/seqid=0 2024-11-16T14:15:36,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742117_1293 (size=12301) 2024-11-16T14:15:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T14:15:36,895 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:36,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:36,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-16T14:15:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:15:36,898 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:36,899 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:36,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:15:37,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:15:37,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:37,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766597175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766597175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766597176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766597176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:37,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766597176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:37,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:15:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:15:37,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/f18fe92a410e434c88d69d6fb2d2948f 2024-11-16T14:15:37,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/f3637db4491040b88f5d734cfe8e3b41 is 50, key is test_row_0/C:col10/1731766536472/Put/seqid=0 2024-11-16T14:15:37,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742118_1294 (size=12301) 2024-11-16T14:15:37,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:15:37,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:15:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:15:37,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:15:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:37,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766597686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766597686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:37,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766597686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:37,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766597686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:37,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766597695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:37,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/f3637db4491040b88f5d734cfe8e3b41 2024-11-16T14:15:37,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/4562844198e245c881e0777fbacf3eb1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/4562844198e245c881e0777fbacf3eb1 2024-11-16T14:15:37,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/4562844198e245c881e0777fbacf3eb1, entries=250, sequenceid=511, filesize=16.8 K 2024-11-16T14:15:37,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/f18fe92a410e434c88d69d6fb2d2948f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/f18fe92a410e434c88d69d6fb2d2948f 2024-11-16T14:15:37,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/f18fe92a410e434c88d69d6fb2d2948f, entries=150, sequenceid=511, filesize=12.0 K 2024-11-16T14:15:37,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/f3637db4491040b88f5d734cfe8e3b41 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/f3637db4491040b88f5d734cfe8e3b41 2024-11-16T14:15:37,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/f3637db4491040b88f5d734cfe8e3b41, entries=150, sequenceid=511, filesize=12.0 K 2024-11-16T14:15:37,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 5bda05ae08ffa563beaf32369a8bf2d2 in 1250ms, sequenceid=511, compaction requested=true 2024-11-16T14:15:37,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:37,723 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:37,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:37,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:37,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:37,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:37,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:37,724 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:37,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:37,724 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:37,724 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/A is initiating minor compaction (all files) 2024-11-16T14:15:37,724 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:37,724 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/A in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,724 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/B is initiating minor compaction (all files) 2024-11-16T14:15:37,725 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/B in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,725 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/3644d418fb4d46d38f5743caa17f04d4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f97d0ba766be4d0db873e649e60490b1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/4562844198e245c881e0777fbacf3eb1] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=41.4 K 2024-11-16T14:15:37,725 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/ec00d167a93041bcbc4970d4051553c2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/192145a293414edca02adcec58eb1cf2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/f18fe92a410e434c88d69d6fb2d2948f] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.6 K 2024-11-16T14:15:37,725 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ec00d167a93041bcbc4970d4051553c2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1731766534216 2024-11-16T14:15:37,725 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3644d418fb4d46d38f5743caa17f04d4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1731766534216 2024-11-16T14:15:37,725 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f97d0ba766be4d0db873e649e60490b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731766534311 2024-11-16T14:15:37,725 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 192145a293414edca02adcec58eb1cf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731766534311 2024-11-16T14:15:37,726 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4562844198e245c881e0777fbacf3eb1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731766536466 2024-11-16T14:15:37,726 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f18fe92a410e434c88d69d6fb2d2948f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731766536466 2024-11-16T14:15:37,734 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#B#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:37,734 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#A#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:37,734 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f38829d0654c4ea4bc25f05c9fa689cf is 50, key is test_row_0/A:col10/1731766536472/Put/seqid=0 2024-11-16T14:15:37,734 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/fbbd094e24f74a99acc49857fab6dcc4 is 50, key is test_row_0/B:col10/1731766536472/Put/seqid=0 2024-11-16T14:15:37,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742120_1296 (size=12983) 2024-11-16T14:15:37,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742119_1295 (size=12983) 2024-11-16T14:15:37,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:15:37,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:37,815 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-16T14:15:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:37,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/23f509f643744d78a30b913436656a33 is 50, key is test_row_0/A:col10/1731766536513/Put/seqid=0 2024-11-16T14:15:37,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742121_1297 (size=12301) 2024-11-16T14:15:37,824 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/23f509f643744d78a30b913436656a33 2024-11-16T14:15:37,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/4c4055b4a3644df780ec0968710aca26 is 50, key is test_row_0/B:col10/1731766536513/Put/seqid=0 2024-11-16T14:15:37,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742122_1298 (size=12301) 2024-11-16T14:15:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:15:38,143 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f38829d0654c4ea4bc25f05c9fa689cf as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f38829d0654c4ea4bc25f05c9fa689cf 2024-11-16T14:15:38,143 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/fbbd094e24f74a99acc49857fab6dcc4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/fbbd094e24f74a99acc49857fab6dcc4 2024-11-16T14:15:38,147 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/B of 5bda05ae08ffa563beaf32369a8bf2d2 into fbbd094e24f74a99acc49857fab6dcc4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:38,147 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:38,147 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/B, priority=13, startTime=1731766537723; duration=0sec 2024-11-16T14:15:38,147 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:38,147 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:B 2024-11-16T14:15:38,147 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/A of 5bda05ae08ffa563beaf32369a8bf2d2 into f38829d0654c4ea4bc25f05c9fa689cf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:38,147 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:38,147 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:38,147 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/A, priority=13, startTime=1731766537723; duration=0sec 2024-11-16T14:15:38,147 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:38,147 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:A 2024-11-16T14:15:38,148 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:38,149 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/C is initiating minor compaction (all files) 2024-11-16T14:15:38,149 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/C in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:38,149 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/17823036a7154c208898608b37fe08bd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/6296c15d265442a7a4840db17181af4e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/f3637db4491040b88f5d734cfe8e3b41] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.6 K 2024-11-16T14:15:38,149 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 17823036a7154c208898608b37fe08bd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1731766534216 2024-11-16T14:15:38,150 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6296c15d265442a7a4840db17181af4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731766534311 2024-11-16T14:15:38,150 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f3637db4491040b88f5d734cfe8e3b41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731766536466 2024-11-16T14:15:38,164 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#C#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:38,164 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/7b477bf1df3d4a0c869faa3e0341a557 is 50, key is test_row_0/C:col10/1731766536472/Put/seqid=0 2024-11-16T14:15:38,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742123_1299 (size=12983) 2024-11-16T14:15:38,236 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/4c4055b4a3644df780ec0968710aca26 2024-11-16T14:15:38,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/ed120a7569a04b789a343ee04bb10ff3 is 50, key is test_row_0/C:col10/1731766536513/Put/seqid=0 2024-11-16T14:15:38,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742124_1300 (size=12301) 2024-11-16T14:15:38,256 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/ed120a7569a04b789a343ee04bb10ff3 2024-11-16T14:15:38,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/23f509f643744d78a30b913436656a33 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/23f509f643744d78a30b913436656a33 2024-11-16T14:15:38,277 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/23f509f643744d78a30b913436656a33, entries=150, sequenceid=547, filesize=12.0 K 2024-11-16T14:15:38,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/4c4055b4a3644df780ec0968710aca26 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/4c4055b4a3644df780ec0968710aca26 2024-11-16T14:15:38,282 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/4c4055b4a3644df780ec0968710aca26, entries=150, sequenceid=547, filesize=12.0 K 2024-11-16T14:15:38,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/ed120a7569a04b789a343ee04bb10ff3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/ed120a7569a04b789a343ee04bb10ff3 2024-11-16T14:15:38,286 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/ed120a7569a04b789a343ee04bb10ff3, entries=150, sequenceid=547, filesize=12.0 K 2024-11-16T14:15:38,287 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for 5bda05ae08ffa563beaf32369a8bf2d2 in 472ms, sequenceid=547, compaction requested=false 2024-11-16T14:15:38,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:38,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:38,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-16T14:15:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-16T14:15:38,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-16T14:15:38,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3890 sec 2024-11-16T14:15:38,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3930 sec 2024-11-16T14:15:38,604 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/7b477bf1df3d4a0c869faa3e0341a557 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/7b477bf1df3d4a0c869faa3e0341a557 2024-11-16T14:15:38,618 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/C of 5bda05ae08ffa563beaf32369a8bf2d2 into 7b477bf1df3d4a0c869faa3e0341a557(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:38,618 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:38,618 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/C, priority=13, startTime=1731766537723; duration=0sec 2024-11-16T14:15:38,618 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:38,618 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:C 2024-11-16T14:15:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:38,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:15:38,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:38,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:38,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:38,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:38,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:38,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:38,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/6e4ca7cf2a4c44e093b3a1104adbf8d6 is 50, key is test_row_0/A:col10/1731766538700/Put/seqid=0 2024-11-16T14:15:38,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742125_1301 (size=14741) 2024-11-16T14:15:38,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/6e4ca7cf2a4c44e093b3a1104adbf8d6 2024-11-16T14:15:38,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766538717, max timestamp=1731766538724Num uniques 859; 2024-11-16T14:15:38,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:15:38,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766538717, max timestamp=1731766538724Num uniques 859; 2024-11-16T14:15:38,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:15:38,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731766538719, max timestamp=1731766538724Num uniques 859; 2024-11-16T14:15:38,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-16T14:15:38,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/ca65448ddf4746e4a4d53d9f9c2129ab is 50, key is test_row_0/B:col10/1731766538700/Put/seqid=0 2024-11-16T14:15:38,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742126_1302 (size=12301) 2024-11-16T14:15:38,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/ca65448ddf4746e4a4d53d9f9c2129ab 2024-11-16T14:15:38,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766598742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:38,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/53136f6107bd4241a2a9cd8b1f17be24 is 50, key is test_row_0/C:col10/1731766538700/Put/seqid=0 2024-11-16T14:15:38,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766598743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:38,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:38,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766598744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:38,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:38,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:38,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766598746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:38,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742127_1303 (size=12301) 2024-11-16T14:15:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:38,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766598748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:38,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766598856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:38,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766598856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766598856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766598856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:38,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766598865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:15:39,045 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:39,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-16T14:15:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T14:15:39,047 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:39,048 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:39,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:39,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766599065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766599065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766599065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766599066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766599076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T14:15:39,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/53136f6107bd4241a2a9cd8b1f17be24 2024-11-16T14:15:39,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/6e4ca7cf2a4c44e093b3a1104adbf8d6 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/6e4ca7cf2a4c44e093b3a1104adbf8d6 2024-11-16T14:15:39,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/6e4ca7cf2a4c44e093b3a1104adbf8d6, entries=200, sequenceid=560, filesize=14.4 K 2024-11-16T14:15:39,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/ca65448ddf4746e4a4d53d9f9c2129ab as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/ca65448ddf4746e4a4d53d9f9c2129ab 2024-11-16T14:15:39,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/ca65448ddf4746e4a4d53d9f9c2129ab, entries=150, sequenceid=560, filesize=12.0 K 2024-11-16T14:15:39,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/53136f6107bd4241a2a9cd8b1f17be24 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/53136f6107bd4241a2a9cd8b1f17be24 2024-11-16T14:15:39,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/53136f6107bd4241a2a9cd8b1f17be24, entries=150, sequenceid=560, filesize=12.0 K 2024-11-16T14:15:39,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.02 KB/139285 for 5bda05ae08ffa563beaf32369a8bf2d2 in 467ms, sequenceid=560, compaction requested=true 2024-11-16T14:15:39,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:39,171 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:39,171 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:39,171 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:39,171 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:39,172 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/A is initiating minor compaction (all files) 2024-11-16T14:15:39,172 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/B is initiating minor compaction (all files) 2024-11-16T14:15:39,172 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/A in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:39,172 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/B in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:39,172 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/fbbd094e24f74a99acc49857fab6dcc4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/4c4055b4a3644df780ec0968710aca26, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/ca65448ddf4746e4a4d53d9f9c2129ab] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.7 K 2024-11-16T14:15:39,172 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f38829d0654c4ea4bc25f05c9fa689cf, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/23f509f643744d78a30b913436656a33, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/6e4ca7cf2a4c44e093b3a1104adbf8d6] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=39.1 K 2024-11-16T14:15:39,172 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f38829d0654c4ea4bc25f05c9fa689cf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731766536466 2024-11-16T14:15:39,172 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting fbbd094e24f74a99acc49857fab6dcc4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731766536466 2024-11-16T14:15:39,173 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23f509f643744d78a30b913436656a33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1731766536513 2024-11-16T14:15:39,173 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c4055b4a3644df780ec0968710aca26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1731766536513 2024-11-16T14:15:39,173 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e4ca7cf2a4c44e093b3a1104adbf8d6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1731766538697 2024-11-16T14:15:39,173 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ca65448ddf4746e4a4d53d9f9c2129ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1731766538697 2024-11-16T14:15:39,180 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:39,180 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/39b159d181ae4a51a8a8c80fedf550ed is 50, key is test_row_0/B:col10/1731766538700/Put/seqid=0 2024-11-16T14:15:39,187 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:39,188 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/0fadaec673a1446ebf4b41551acbf9f7 is 50, key is test_row_0/A:col10/1731766538700/Put/seqid=0 2024-11-16T14:15:39,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T14:15:39,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:39,200 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=136.02 KB heapSize=384.46 KB 2024-11-16T14:15:39,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:39,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:39,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:39,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:39,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:39,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:39,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742128_1304 (size=13085) 2024-11-16T14:15:39,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/a63f541a3e7248a5ab459f36ead8f288 is 50, key is test_row_0/A:col10/1731766538745/Put/seqid=0 2024-11-16T14:15:39,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742129_1305 (size=13085) 2024-11-16T14:15:39,232 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/0fadaec673a1446ebf4b41551acbf9f7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/0fadaec673a1446ebf4b41551acbf9f7 2024-11-16T14:15:39,241 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/A of 5bda05ae08ffa563beaf32369a8bf2d2 into 0fadaec673a1446ebf4b41551acbf9f7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:39,241 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:39,241 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/A, priority=13, startTime=1731766539171; duration=0sec 2024-11-16T14:15:39,242 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:39,242 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:A 2024-11-16T14:15:39,242 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:39,243 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:39,243 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/C is initiating minor compaction (all files) 2024-11-16T14:15:39,243 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/C in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:39,243 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/7b477bf1df3d4a0c869faa3e0341a557, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/ed120a7569a04b789a343ee04bb10ff3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/53136f6107bd4241a2a9cd8b1f17be24] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.7 K 2024-11-16T14:15:39,243 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b477bf1df3d4a0c869faa3e0341a557, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731766536466 2024-11-16T14:15:39,244 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed120a7569a04b789a343ee04bb10ff3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1731766536513 2024-11-16T14:15:39,245 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53136f6107bd4241a2a9cd8b1f17be24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1731766538697 2024-11-16T14:15:39,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742130_1306 (size=12301) 2024-11-16T14:15:39,254 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#C#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:39,254 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/508965ffe52540a9a33d0269da3e4ab0 is 50, key is test_row_0/C:col10/1731766538700/Put/seqid=0 2024-11-16T14:15:39,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742131_1307 (size=13085) 2024-11-16T14:15:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T14:15:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:39,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:39,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766599382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766599383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766599386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766599386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766599387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766599496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766599496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766599496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766599496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,606 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/39b159d181ae4a51a8a8c80fedf550ed as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/39b159d181ae4a51a8a8c80fedf550ed 2024-11-16T14:15:39,610 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/B of 5bda05ae08ffa563beaf32369a8bf2d2 into 39b159d181ae4a51a8a8c80fedf550ed(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:39,610 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:39,610 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/B, priority=13, startTime=1731766539171; duration=0sec 2024-11-16T14:15:39,610 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:39,611 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:B 2024-11-16T14:15:39,648 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/a63f541a3e7248a5ab459f36ead8f288 2024-11-16T14:15:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/1e32474ce21e4d3d95f24152ee0c8d72 is 50, key is test_row_0/B:col10/1731766538745/Put/seqid=0 2024-11-16T14:15:39,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742132_1308 (size=12301) 2024-11-16T14:15:39,665 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/508965ffe52540a9a33d0269da3e4ab0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/508965ffe52540a9a33d0269da3e4ab0 2024-11-16T14:15:39,669 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/C of 5bda05ae08ffa563beaf32369a8bf2d2 into 508965ffe52540a9a33d0269da3e4ab0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:39,669 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:39,669 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/C, priority=13, startTime=1731766539171; duration=0sec 2024-11-16T14:15:39,669 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:39,669 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:C 2024-11-16T14:15:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T14:15:39,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766599706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766599706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766599706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766599706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:39,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766599896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766600016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766600016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766600016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766600016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,060 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/1e32474ce21e4d3d95f24152ee0c8d72 2024-11-16T14:15:40,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/e301d159630e43e297f0c3ef56609cb7 is 50, key is test_row_0/C:col10/1731766538745/Put/seqid=0 2024-11-16T14:15:40,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742133_1309 (size=12301) 2024-11-16T14:15:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T14:15:40,486 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/e301d159630e43e297f0c3ef56609cb7 2024-11-16T14:15:40,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/a63f541a3e7248a5ab459f36ead8f288 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/a63f541a3e7248a5ab459f36ead8f288 2024-11-16T14:15:40,496 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/a63f541a3e7248a5ab459f36ead8f288, entries=150, sequenceid=596, filesize=12.0 K 2024-11-16T14:15:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/1e32474ce21e4d3d95f24152ee0c8d72 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/1e32474ce21e4d3d95f24152ee0c8d72 2024-11-16T14:15:40,501 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/1e32474ce21e4d3d95f24152ee0c8d72, entries=150, sequenceid=596, filesize=12.0 K 2024-11-16T14:15:40,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/e301d159630e43e297f0c3ef56609cb7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/e301d159630e43e297f0c3ef56609cb7 2024-11-16T14:15:40,507 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/e301d159630e43e297f0c3ef56609cb7, entries=150, sequenceid=596, filesize=12.0 K 2024-11-16T14:15:40,508 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~136.02 KB/139285, heapSize ~384.41 KB/393640, currentSize=46.96 KB/48090 for 5bda05ae08ffa563beaf32369a8bf2d2 in 1308ms, sequenceid=596, compaction requested=false 2024-11-16T14:15:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-16T14:15:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-16T14:15:40,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-16T14:15:40,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4620 sec 2024-11-16T14:15:40,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4670 sec 2024-11-16T14:15:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:40,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:15:40,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:40,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:40,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:40,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:40,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:40,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:40,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f16972a9be86447f916b97ab02423c0e is 50, key is test_row_0/A:col10/1731766540527/Put/seqid=0 2024-11-16T14:15:40,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742134_1310 (size=14741) 2024-11-16T14:15:40,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766600552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766600559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766600559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766600559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766600665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766600665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766600676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766600676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766600875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766600875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766600886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766600886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766600905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:40,909 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in 5bda05ae08ffa563beaf32369a8bf2d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:15:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:40,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:40,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f16972a9be86447f916b97ab02423c0e 2024-11-16T14:15:40,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/a912dbb1385e47eb8a207d405eb86ad7 is 50, key is test_row_0/B:col10/1731766540527/Put/seqid=0 2024-11-16T14:15:40,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742135_1311 (size=12301) 2024-11-16T14:15:41,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766601185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:41,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766601185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:41,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T14:15:41,196 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:41,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-16T14:15:41,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T14:15:41,198 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:41,199 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:41,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:41,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:41,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766601196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766601196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T14:15:41,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T14:15:41,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:41,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:41,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:41,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:41,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:41,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/a912dbb1385e47eb8a207d405eb86ad7 2024-11-16T14:15:41,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/ec8527fa15564ba68f7807d6196a808e is 50, key is test_row_0/C:col10/1731766540527/Put/seqid=0 2024-11-16T14:15:41,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742136_1312 (size=12301) 2024-11-16T14:15:41,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T14:15:41,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:41,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:41,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:41,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:41,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T14:15:41,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T14:15:41,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:41,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:41,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:41,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:41,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:41,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766601695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766601695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:41,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766601706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766601706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:41,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/ec8527fa15564ba68f7807d6196a808e 2024-11-16T14:15:41,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/f16972a9be86447f916b97ab02423c0e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f16972a9be86447f916b97ab02423c0e 2024-11-16T14:15:41,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f16972a9be86447f916b97ab02423c0e, entries=200, sequenceid=612, filesize=14.4 K 2024-11-16T14:15:41,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/a912dbb1385e47eb8a207d405eb86ad7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/a912dbb1385e47eb8a207d405eb86ad7 2024-11-16T14:15:41,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/a912dbb1385e47eb8a207d405eb86ad7, entries=150, sequenceid=612, filesize=12.0 K 2024-11-16T14:15:41,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/ec8527fa15564ba68f7807d6196a808e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/ec8527fa15564ba68f7807d6196a808e 2024-11-16T14:15:41,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/ec8527fa15564ba68f7807d6196a808e, entries=150, sequenceid=612, filesize=12.0 K 2024-11-16T14:15:41,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5bda05ae08ffa563beaf32369a8bf2d2 in 1263ms, sequenceid=612, compaction requested=true 2024-11-16T14:15:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:41,792 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:41,792 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:41,800 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:41,800 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/B is initiating minor compaction (all files) 2024-11-16T14:15:41,800 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/B in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:41,800 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/39b159d181ae4a51a8a8c80fedf550ed, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/1e32474ce21e4d3d95f24152ee0c8d72, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/a912dbb1385e47eb8a207d405eb86ad7] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.8 K 2024-11-16T14:15:41,801 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:41,801 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/A is initiating minor compaction (all files) 2024-11-16T14:15:41,801 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/A in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:41,801 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/0fadaec673a1446ebf4b41551acbf9f7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/a63f541a3e7248a5ab459f36ead8f288, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f16972a9be86447f916b97ab02423c0e] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=39.2 K 2024-11-16T14:15:41,801 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 39b159d181ae4a51a8a8c80fedf550ed, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1731766538697 2024-11-16T14:15:41,801 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fadaec673a1446ebf4b41551acbf9f7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1731766538697 2024-11-16T14:15:41,802 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e32474ce21e4d3d95f24152ee0c8d72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=596, earliestPutTs=1731766538742 2024-11-16T14:15:41,802 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting a63f541a3e7248a5ab459f36ead8f288, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=596, earliestPutTs=1731766538742 2024-11-16T14:15:41,802 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting a912dbb1385e47eb8a207d405eb86ad7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731766539380 2024-11-16T14:15:41,802 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f16972a9be86447f916b97ab02423c0e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731766539380 2024-11-16T14:15:41,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T14:15:41,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:41,812 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:15:41,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:41,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:41,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:41,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:41,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:41,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:41,812 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#A#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:41,813 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/cbcbe7a024a94f07af85a7f42a6c1dd4 is 50, key is test_row_0/A:col10/1731766540527/Put/seqid=0 2024-11-16T14:15:41,822 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#B#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:41,822 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/babe9f263afe4c66b1bfe88d33d244a2 is 50, key is test_row_0/B:col10/1731766540527/Put/seqid=0 2024-11-16T14:15:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T14:15:41,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/dec3b9fa777a4011a1e45999f7b5f94d is 50, key is test_row_0/A:col10/1731766540543/Put/seqid=0 2024-11-16T14:15:41,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742137_1313 (size=13187) 2024-11-16T14:15:41,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742139_1315 (size=12301) 2024-11-16T14:15:41,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742138_1314 (size=13187) 2024-11-16T14:15:42,247 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/cbcbe7a024a94f07af85a7f42a6c1dd4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/cbcbe7a024a94f07af85a7f42a6c1dd4 2024-11-16T14:15:42,252 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/A of 5bda05ae08ffa563beaf32369a8bf2d2 into cbcbe7a024a94f07af85a7f42a6c1dd4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:42,252 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:42,252 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/A, priority=13, startTime=1731766541792; duration=0sec 2024-11-16T14:15:42,252 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:42,252 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:A 2024-11-16T14:15:42,252 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:42,253 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:42,253 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/C is initiating minor compaction (all files) 2024-11-16T14:15:42,253 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/C in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:42,253 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/508965ffe52540a9a33d0269da3e4ab0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/e301d159630e43e297f0c3ef56609cb7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/ec8527fa15564ba68f7807d6196a808e] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.8 K 2024-11-16T14:15:42,253 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/dec3b9fa777a4011a1e45999f7b5f94d 2024-11-16T14:15:42,254 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 508965ffe52540a9a33d0269da3e4ab0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1731766538697 2024-11-16T14:15:42,254 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e301d159630e43e297f0c3ef56609cb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=596, earliestPutTs=1731766538742 2024-11-16T14:15:42,255 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec8527fa15564ba68f7807d6196a808e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731766539380 2024-11-16T14:15:42,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/0b552526819c49ad854dad8cd58f303b is 50, key is test_row_0/B:col10/1731766540543/Put/seqid=0 2024-11-16T14:15:42,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742140_1316 (size=12301) 2024-11-16T14:15:42,274 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#C#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:42,274 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/da7e56971e2a4670857ac5c680c8a057 is 50, key is test_row_0/C:col10/1731766540527/Put/seqid=0 2024-11-16T14:15:42,286 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/babe9f263afe4c66b1bfe88d33d244a2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/babe9f263afe4c66b1bfe88d33d244a2 2024-11-16T14:15:42,292 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/B of 5bda05ae08ffa563beaf32369a8bf2d2 into babe9f263afe4c66b1bfe88d33d244a2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:42,292 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:42,292 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/B, priority=13, startTime=1731766541792; duration=0sec 2024-11-16T14:15:42,292 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:42,292 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:B 2024-11-16T14:15:42,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742141_1317 (size=13187) 2024-11-16T14:15:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T14:15:42,671 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/0b552526819c49ad854dad8cd58f303b 2024-11-16T14:15:42,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/661a884e897747cb992f43dc7d23ce47 is 50, key is test_row_0/C:col10/1731766540543/Put/seqid=0 2024-11-16T14:15:42,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742142_1318 (size=12301) 2024-11-16T14:15:42,688 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/661a884e897747cb992f43dc7d23ce47 2024-11-16T14:15:42,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/dec3b9fa777a4011a1e45999f7b5f94d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/dec3b9fa777a4011a1e45999f7b5f94d 2024-11-16T14:15:42,698 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/dec3b9fa777a4011a1e45999f7b5f94d, entries=150, sequenceid=631, filesize=12.0 K 2024-11-16T14:15:42,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/0b552526819c49ad854dad8cd58f303b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0b552526819c49ad854dad8cd58f303b 2024-11-16T14:15:42,701 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/da7e56971e2a4670857ac5c680c8a057 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/da7e56971e2a4670857ac5c680c8a057 2024-11-16T14:15:42,707 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0b552526819c49ad854dad8cd58f303b, entries=150, sequenceid=631, filesize=12.0 K 2024-11-16T14:15:42,708 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/C of 5bda05ae08ffa563beaf32369a8bf2d2 into da7e56971e2a4670857ac5c680c8a057(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:42,708 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:42,708 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/C, priority=13, startTime=1731766541792; duration=0sec 2024-11-16T14:15:42,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:42,708 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:42,708 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:C 2024-11-16T14:15:42,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-16T14:15:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/661a884e897747cb992f43dc7d23ce47 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/661a884e897747cb992f43dc7d23ce47 2024-11-16T14:15:42,724 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/661a884e897747cb992f43dc7d23ce47, entries=150, sequenceid=631, filesize=12.0 K 2024-11-16T14:15:42,725 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=53.67 KB/54960 for 5bda05ae08ffa563beaf32369a8bf2d2 in 914ms, sequenceid=631, compaction requested=false 2024-11-16T14:15:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-16T14:15:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-16T14:15:42,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-16T14:15:42,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5280 sec 2024-11-16T14:15:42,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5330 sec 2024-11-16T14:15:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:42,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:15:42,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:42,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:42,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:42,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:42,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:42,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:42,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/ceb4e0cb5f5242e09a0f7aa89218e697 is 50, key is test_row_0/A:col10/1731766542730/Put/seqid=0 2024-11-16T14:15:42,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742143_1319 (size=12301) 2024-11-16T14:15:42,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766602754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:42,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766602755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:42,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766602756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766602757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766602865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:42,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766602865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:42,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766602866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766602866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:42,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36966 deadline: 1731766602915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:42,918 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in 5bda05ae08ffa563beaf32369a8bf2d2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:15:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766603075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766603076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766603076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766603076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/ceb4e0cb5f5242e09a0f7aa89218e697 2024-11-16T14:15:43,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/bf14cfa060514a93901b7a8cee3f280b is 50, key is test_row_0/B:col10/1731766542730/Put/seqid=0 2024-11-16T14:15:43,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742144_1320 (size=12301) 2024-11-16T14:15:43,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/bf14cfa060514a93901b7a8cee3f280b 2024-11-16T14:15:43,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/671300e5bedf40e39591cdb57ed97bf0 is 50, key is test_row_0/C:col10/1731766542730/Put/seqid=0 2024-11-16T14:15:43,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742145_1321 (size=12301) 2024-11-16T14:15:43,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/671300e5bedf40e39591cdb57ed97bf0 2024-11-16T14:15:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/ceb4e0cb5f5242e09a0f7aa89218e697 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/ceb4e0cb5f5242e09a0f7aa89218e697 2024-11-16T14:15:43,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/ceb4e0cb5f5242e09a0f7aa89218e697, entries=150, sequenceid=647, filesize=12.0 K 2024-11-16T14:15:43,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/bf14cfa060514a93901b7a8cee3f280b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/bf14cfa060514a93901b7a8cee3f280b 2024-11-16T14:15:43,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/bf14cfa060514a93901b7a8cee3f280b, entries=150, sequenceid=647, filesize=12.0 K 2024-11-16T14:15:43,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/671300e5bedf40e39591cdb57ed97bf0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/671300e5bedf40e39591cdb57ed97bf0 2024-11-16T14:15:43,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/671300e5bedf40e39591cdb57ed97bf0, entries=150, sequenceid=647, filesize=12.0 K 2024-11-16T14:15:43,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5bda05ae08ffa563beaf32369a8bf2d2 in 466ms, sequenceid=647, compaction requested=true 2024-11-16T14:15:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:43,198 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:43,198 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5bda05ae08ffa563beaf32369a8bf2d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:43,199 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:43,199 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/A is initiating minor compaction (all files) 2024-11-16T14:15:43,199 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/A in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:43,200 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/cbcbe7a024a94f07af85a7f42a6c1dd4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/dec3b9fa777a4011a1e45999f7b5f94d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/ceb4e0cb5f5242e09a0f7aa89218e697] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.9 K 2024-11-16T14:15:43,200 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:43,200 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/B is initiating minor compaction (all files) 2024-11-16T14:15:43,200 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbcbe7a024a94f07af85a7f42a6c1dd4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731766539380 2024-11-16T14:15:43,200 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/B in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:43,200 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/babe9f263afe4c66b1bfe88d33d244a2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0b552526819c49ad854dad8cd58f303b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/bf14cfa060514a93901b7a8cee3f280b] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.9 K 2024-11-16T14:15:43,200 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting dec3b9fa777a4011a1e45999f7b5f94d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731766540543 2024-11-16T14:15:43,200 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceb4e0cb5f5242e09a0f7aa89218e697, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1731766542720 2024-11-16T14:15:43,200 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting babe9f263afe4c66b1bfe88d33d244a2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731766539380 2024-11-16T14:15:43,201 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b552526819c49ad854dad8cd58f303b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731766540543 2024-11-16T14:15:43,202 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting bf14cfa060514a93901b7a8cee3f280b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1731766542720 2024-11-16T14:15:43,209 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#A#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:43,210 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#B#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:43,210 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/a0d8f28e60f941c387b07761a1693812 is 50, key is test_row_0/A:col10/1731766542730/Put/seqid=0 2024-11-16T14:15:43,212 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/8c701313edd147d895d291aaed8a83ea is 50, key is test_row_0/B:col10/1731766542730/Put/seqid=0 2024-11-16T14:15:43,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742146_1322 (size=13289) 2024-11-16T14:15:43,241 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/a0d8f28e60f941c387b07761a1693812 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/a0d8f28e60f941c387b07761a1693812 2024-11-16T14:15:43,265 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/A of 5bda05ae08ffa563beaf32369a8bf2d2 into a0d8f28e60f941c387b07761a1693812(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:43,265 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:43,265 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/A, priority=13, startTime=1731766543198; duration=0sec 2024-11-16T14:15:43,265 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:43,265 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:A 2024-11-16T14:15:43,265 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:43,266 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:43,266 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 5bda05ae08ffa563beaf32369a8bf2d2/C is initiating minor compaction (all files) 2024-11-16T14:15:43,266 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5bda05ae08ffa563beaf32369a8bf2d2/C in TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:43,266 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/da7e56971e2a4670857ac5c680c8a057, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/661a884e897747cb992f43dc7d23ce47, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/671300e5bedf40e39591cdb57ed97bf0] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp, totalSize=36.9 K 2024-11-16T14:15:43,267 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting da7e56971e2a4670857ac5c680c8a057, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731766539380 2024-11-16T14:15:43,267 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 661a884e897747cb992f43dc7d23ce47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731766540543 2024-11-16T14:15:43,267 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 671300e5bedf40e39591cdb57ed97bf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1731766542720 2024-11-16T14:15:43,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742147_1323 (size=13289) 2024-11-16T14:15:43,281 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/8c701313edd147d895d291aaed8a83ea as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/8c701313edd147d895d291aaed8a83ea 2024-11-16T14:15:43,283 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5bda05ae08ffa563beaf32369a8bf2d2#C#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:43,283 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/bb83c17b94fc45d08d589d0a00659a4b is 50, key is test_row_0/C:col10/1731766542730/Put/seqid=0 2024-11-16T14:15:43,299 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/B of 5bda05ae08ffa563beaf32369a8bf2d2 into 8c701313edd147d895d291aaed8a83ea(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:43,299 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:43,299 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/B, priority=13, startTime=1731766543198; duration=0sec 2024-11-16T14:15:43,299 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:43,299 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:B 2024-11-16T14:15:43,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742148_1324 (size=13289) 2024-11-16T14:15:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T14:15:43,346 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:43,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-16T14:15:43,347 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:15:43,348 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:43,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:43,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-16T14:15:43,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:43,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:43,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:43,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:43,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:43,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:43,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/8ebb0b0cc3864043a5b237874f9c7c00 is 50, key is test_row_0/A:col10/1731766542756/Put/seqid=0 2024-11-16T14:15:43,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742149_1325 (size=12301) 2024-11-16T14:15:43,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766603400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766603400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766603404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766603406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:15:43,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T14:15:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:43,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:43,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766603515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766603515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766603515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766603515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T14:15:43,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:43,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:43,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:43,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:43,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:15:43,718 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/bb83c17b94fc45d08d589d0a00659a4b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/bb83c17b94fc45d08d589d0a00659a4b 2024-11-16T14:15:43,722 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5bda05ae08ffa563beaf32369a8bf2d2/C of 5bda05ae08ffa563beaf32369a8bf2d2 into bb83c17b94fc45d08d589d0a00659a4b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:43,722 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:43,722 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., storeName=5bda05ae08ffa563beaf32369a8bf2d2/C, priority=13, startTime=1731766543198; duration=0sec 2024-11-16T14:15:43,722 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:43,722 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5bda05ae08ffa563beaf32369a8bf2d2:C 2024-11-16T14:15:43,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766603725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766603725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766603725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766603726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:43,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=670 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/8ebb0b0cc3864043a5b237874f9c7c00 2024-11-16T14:15:43,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T14:15:43,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:43,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:43,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:43,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:43,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:43,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/c1bbac8a40de4bb8bc47ce6cb8a54e73 is 50, key is test_row_0/B:col10/1731766542756/Put/seqid=0 2024-11-16T14:15:43,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742150_1326 (size=12301) 2024-11-16T14:15:43,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=670 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/c1bbac8a40de4bb8bc47ce6cb8a54e73 2024-11-16T14:15:43,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/05826e62b33142f9b040337d4d8a3d83 is 50, key is test_row_0/C:col10/1731766542756/Put/seqid=0 2024-11-16T14:15:43,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742151_1327 (size=12301) 2024-11-16T14:15:43,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T14:15:43,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:43,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:43,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:43,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:43,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:15:44,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36986 deadline: 1731766604035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:44,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36972 deadline: 1731766604040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:44,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36940 deadline: 1731766604040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:44,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36932 deadline: 1731766604040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:15:44,108 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-16T14:15:44,108 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:44,108 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,108 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,108 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:44,109 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-16T14:15:44,109 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:44,109 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,109 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,110 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:44,111 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-16T14:15:44,111 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:44,111 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,111 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,111 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-16T14:15:44,111 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:44,111 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:44,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:44,111 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,112 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-16T14:15:44,112 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:44,112 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,113 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:44,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T14:15:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:44,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:44,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=670 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/05826e62b33142f9b040337d4d8a3d83 2024-11-16T14:15:44,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/8ebb0b0cc3864043a5b237874f9c7c00 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8ebb0b0cc3864043a5b237874f9c7c00 2024-11-16T14:15:44,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8ebb0b0cc3864043a5b237874f9c7c00, entries=150, sequenceid=670, filesize=12.0 K 2024-11-16T14:15:44,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/c1bbac8a40de4bb8bc47ce6cb8a54e73 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c1bbac8a40de4bb8bc47ce6cb8a54e73 2024-11-16T14:15:44,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c1bbac8a40de4bb8bc47ce6cb8a54e73, entries=150, sequenceid=670, filesize=12.0 K 2024-11-16T14:15:44,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/05826e62b33142f9b040337d4d8a3d83 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/05826e62b33142f9b040337d4d8a3d83 2024-11-16T14:15:44,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/05826e62b33142f9b040337d4d8a3d83, entries=150, sequenceid=670, filesize=12.0 K 2024-11-16T14:15:44,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 5bda05ae08ffa563beaf32369a8bf2d2 in 864ms, sequenceid=670, compaction requested=false 2024-11-16T14:15:44,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:44,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T14:15:44,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:44,267 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:15:44,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:44,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:44,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:44,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:44,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:44,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:44,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/b8f3790777bb4ed0b941eaf454f3ac64 is 50, key is test_row_0/A:col10/1731766543404/Put/seqid=0 2024-11-16T14:15:44,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742152_1328 (size=12301) 2024-11-16T14:15:44,294 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/b8f3790777bb4ed0b941eaf454f3ac64 2024-11-16T14:15:44,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/579538999c944a2c99a5e8b99c126a45 is 50, key is test_row_0/B:col10/1731766543404/Put/seqid=0 2024-11-16T14:15:44,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742153_1329 (size=12301) 2024-11-16T14:15:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:15:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:44,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. as already flushing 2024-11-16T14:15:44,548 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-11-16T14:15:44,548 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:44,548 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,548 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:44,561 INFO [Thread-1102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1102. 2024-11-16T14:15:44,561 DEBUG [Thread-1102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:44,561 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,561 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,561 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:44,586 INFO [Thread-1100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1100. 2024-11-16T14:15:44,586 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-11-16T14:15:44,586 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:44,586 DEBUG [Thread-1100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:44,586 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,586 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,586 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,586 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:44,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:44,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:44,705 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/579538999c944a2c99a5e8b99c126a45 2024-11-16T14:15:44,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/2c2b455843b44d0dbae18588e2888ebc is 50, key is test_row_0/C:col10/1731766543404/Put/seqid=0 2024-11-16T14:15:44,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742154_1330 (size=12301) 2024-11-16T14:15:44,757 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/2c2b455843b44d0dbae18588e2888ebc 2024-11-16T14:15:44,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/b8f3790777bb4ed0b941eaf454f3ac64 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/b8f3790777bb4ed0b941eaf454f3ac64 2024-11-16T14:15:44,779 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/b8f3790777bb4ed0b941eaf454f3ac64, entries=150, sequenceid=682, filesize=12.0 K 2024-11-16T14:15:44,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/579538999c944a2c99a5e8b99c126a45 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/579538999c944a2c99a5e8b99c126a45 2024-11-16T14:15:44,795 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/579538999c944a2c99a5e8b99c126a45, entries=150, sequenceid=682, filesize=12.0 K 2024-11-16T14:15:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/2c2b455843b44d0dbae18588e2888ebc as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/2c2b455843b44d0dbae18588e2888ebc 2024-11-16T14:15:44,800 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/2c2b455843b44d0dbae18588e2888ebc, entries=150, sequenceid=682, filesize=12.0 K 2024-11-16T14:15:44,801 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=26.84 KB/27480 for 5bda05ae08ffa563beaf32369a8bf2d2 in 534ms, sequenceid=682, compaction requested=true 2024-11-16T14:15:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 5bda05ae08ffa563beaf32369a8bf2d2: 2024-11-16T14:15:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-16T14:15:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-16T14:15:44,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-16T14:15:44,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4590 sec 2024-11-16T14:15:44,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.4650 sec 2024-11-16T14:15:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:15:45,496 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:46,947 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-11-16T14:15:46,947 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:15:46,947 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:46,947 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:46,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 154 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 186 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 158 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3084 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2983 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3018 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3057 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2953 2024-11-16T14:15:46,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T14:15:46,948 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:15:46,948 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:15:46,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:46,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:46,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:15:46,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T14:15:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T14:15:46,953 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766546953"}]},"ts":"1731766546953"} 2024-11-16T14:15:46,955 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T14:15:46,955 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T14:15:46,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:15:46,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bda05ae08ffa563beaf32369a8bf2d2, UNASSIGN}] 2024-11-16T14:15:46,961 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bda05ae08ffa563beaf32369a8bf2d2, UNASSIGN 2024-11-16T14:15:46,962 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5bda05ae08ffa563beaf32369a8bf2d2, regionState=CLOSING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:46,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bda05ae08ffa563beaf32369a8bf2d2, UNASSIGN because future has completed 2024-11-16T14:15:46,963 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:15:46,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:15:47,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T14:15:47,116 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:47,116 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:15:47,117 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 5bda05ae08ffa563beaf32369a8bf2d2, disabling compactions & flushes 2024-11-16T14:15:47,117 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:47,117 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:47,117 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. after waiting 0 ms 2024-11-16T14:15:47,117 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:47,117 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 5bda05ae08ffa563beaf32369a8bf2d2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T14:15:47,117 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=A 2024-11-16T14:15:47,117 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:47,117 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=B 2024-11-16T14:15:47,118 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:47,118 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5bda05ae08ffa563beaf32369a8bf2d2, store=C 2024-11-16T14:15:47,118 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:47,131 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/79549a23a3784b53b53374b820ba2ea5 is 50, key is test_row_0/A:col10/1731766544546/Put/seqid=0 2024-11-16T14:15:47,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742155_1331 (size=12301) 2024-11-16T14:15:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T14:15:47,547 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/79549a23a3784b53b53374b820ba2ea5 2024-11-16T14:15:47,561 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/c3dac83154364dc68df67bf79d746e9d is 50, key is test_row_0/B:col10/1731766544546/Put/seqid=0 2024-11-16T14:15:47,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742156_1332 (size=12301) 2024-11-16T14:15:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T14:15:47,966 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/c3dac83154364dc68df67bf79d746e9d 2024-11-16T14:15:47,977 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/a1c5597797564f22b7b9a5cde5c1f106 is 50, key is test_row_0/C:col10/1731766544546/Put/seqid=0 2024-11-16T14:15:47,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742157_1333 (size=12301) 2024-11-16T14:15:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T14:15:48,381 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/a1c5597797564f22b7b9a5cde5c1f106 2024-11-16T14:15:48,386 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/A/79549a23a3784b53b53374b820ba2ea5 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/79549a23a3784b53b53374b820ba2ea5 2024-11-16T14:15:48,392 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/79549a23a3784b53b53374b820ba2ea5, entries=150, sequenceid=690, filesize=12.0 K 2024-11-16T14:15:48,393 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/B/c3dac83154364dc68df67bf79d746e9d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c3dac83154364dc68df67bf79d746e9d 2024-11-16T14:15:48,397 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c3dac83154364dc68df67bf79d746e9d, entries=150, sequenceid=690, filesize=12.0 K 2024-11-16T14:15:48,398 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/.tmp/C/a1c5597797564f22b7b9a5cde5c1f106 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/a1c5597797564f22b7b9a5cde5c1f106 2024-11-16T14:15:48,401 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/a1c5597797564f22b7b9a5cde5c1f106, entries=150, sequenceid=690, filesize=12.0 K 2024-11-16T14:15:48,402 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5bda05ae08ffa563beaf32369a8bf2d2 in 1285ms, sequenceid=690, compaction requested=true 2024-11-16T14:15:48,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/7ad5cf7c3e854ff2a5e5a3cb70ef6a86, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/d2fc574c643145419faaa39d6d522606, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/2f42ef8fbd0d474695b9e6fc33ba2a1e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/aadd48cd40434afe92cd28396ed0fa57, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8b63648d02f046c595055cab0133fb1f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f590652cfbe743329d0704b8cf3f2817, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8ef7173a7e674f8d9bc4f2f0105a71a8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/214101f1f2544109bd8ff747902a4cf2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/bd1a5c718f934360969ea94163000e24, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/19f4e192e86f406ea2790f7f861a78c2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/91efdc9e005f41fd94bf66702dedc2f9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f7c216129aa4417fad9442dd029d757b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/1221779a3ef54311a6b754b0454bf6c6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/ec9fa979677a46c7971394d346462833, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/564265cd975c4028b214ce5fd44a90eb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/c493a1e120e6467dbe17058aad6cff46, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/c22d3659d1b2476087c1eb22cf599920, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/3644d418fb4d46d38f5743caa17f04d4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f97d0ba766be4d0db873e649e60490b1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/4562844198e245c881e0777fbacf3eb1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f38829d0654c4ea4bc25f05c9fa689cf, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/23f509f643744d78a30b913436656a33, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/6e4ca7cf2a4c44e093b3a1104adbf8d6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/0fadaec673a1446ebf4b41551acbf9f7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/a63f541a3e7248a5ab459f36ead8f288, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f16972a9be86447f916b97ab02423c0e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/cbcbe7a024a94f07af85a7f42a6c1dd4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/dec3b9fa777a4011a1e45999f7b5f94d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/ceb4e0cb5f5242e09a0f7aa89218e697] to archive 2024-11-16T14:15:48,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:15:48,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/7ad5cf7c3e854ff2a5e5a3cb70ef6a86 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/7ad5cf7c3e854ff2a5e5a3cb70ef6a86 2024-11-16T14:15:48,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/d2fc574c643145419faaa39d6d522606 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/d2fc574c643145419faaa39d6d522606 2024-11-16T14:15:48,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/2f42ef8fbd0d474695b9e6fc33ba2a1e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/2f42ef8fbd0d474695b9e6fc33ba2a1e 2024-11-16T14:15:48,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/aadd48cd40434afe92cd28396ed0fa57 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/aadd48cd40434afe92cd28396ed0fa57 2024-11-16T14:15:48,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8b63648d02f046c595055cab0133fb1f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8b63648d02f046c595055cab0133fb1f 2024-11-16T14:15:48,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f590652cfbe743329d0704b8cf3f2817 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f590652cfbe743329d0704b8cf3f2817 2024-11-16T14:15:48,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8ef7173a7e674f8d9bc4f2f0105a71a8 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8ef7173a7e674f8d9bc4f2f0105a71a8 2024-11-16T14:15:48,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/214101f1f2544109bd8ff747902a4cf2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/214101f1f2544109bd8ff747902a4cf2 2024-11-16T14:15:48,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/bd1a5c718f934360969ea94163000e24 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/bd1a5c718f934360969ea94163000e24 2024-11-16T14:15:48,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/19f4e192e86f406ea2790f7f861a78c2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/19f4e192e86f406ea2790f7f861a78c2 2024-11-16T14:15:48,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/91efdc9e005f41fd94bf66702dedc2f9 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/91efdc9e005f41fd94bf66702dedc2f9 2024-11-16T14:15:48,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f7c216129aa4417fad9442dd029d757b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f7c216129aa4417fad9442dd029d757b 2024-11-16T14:15:48,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/1221779a3ef54311a6b754b0454bf6c6 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/1221779a3ef54311a6b754b0454bf6c6 2024-11-16T14:15:48,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/ec9fa979677a46c7971394d346462833 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/ec9fa979677a46c7971394d346462833 2024-11-16T14:15:48,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/564265cd975c4028b214ce5fd44a90eb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/564265cd975c4028b214ce5fd44a90eb 2024-11-16T14:15:48,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/c493a1e120e6467dbe17058aad6cff46 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/c493a1e120e6467dbe17058aad6cff46 2024-11-16T14:15:48,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/c22d3659d1b2476087c1eb22cf599920 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/c22d3659d1b2476087c1eb22cf599920 2024-11-16T14:15:48,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/3644d418fb4d46d38f5743caa17f04d4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/3644d418fb4d46d38f5743caa17f04d4 2024-11-16T14:15:48,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f97d0ba766be4d0db873e649e60490b1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f97d0ba766be4d0db873e649e60490b1 2024-11-16T14:15:48,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/4562844198e245c881e0777fbacf3eb1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/4562844198e245c881e0777fbacf3eb1 2024-11-16T14:15:48,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f38829d0654c4ea4bc25f05c9fa689cf to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f38829d0654c4ea4bc25f05c9fa689cf 2024-11-16T14:15:48,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/23f509f643744d78a30b913436656a33 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/23f509f643744d78a30b913436656a33 2024-11-16T14:15:48,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/6e4ca7cf2a4c44e093b3a1104adbf8d6 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/6e4ca7cf2a4c44e093b3a1104adbf8d6 2024-11-16T14:15:48,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/0fadaec673a1446ebf4b41551acbf9f7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/0fadaec673a1446ebf4b41551acbf9f7 2024-11-16T14:15:48,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/a63f541a3e7248a5ab459f36ead8f288 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/a63f541a3e7248a5ab459f36ead8f288 2024-11-16T14:15:48,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f16972a9be86447f916b97ab02423c0e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/f16972a9be86447f916b97ab02423c0e 2024-11-16T14:15:48,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/cbcbe7a024a94f07af85a7f42a6c1dd4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/cbcbe7a024a94f07af85a7f42a6c1dd4 2024-11-16T14:15:48,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/dec3b9fa777a4011a1e45999f7b5f94d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/dec3b9fa777a4011a1e45999f7b5f94d 2024-11-16T14:15:48,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/ceb4e0cb5f5242e09a0f7aa89218e697 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/ceb4e0cb5f5242e09a0f7aa89218e697 2024-11-16T14:15:48,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/515d7c98d82a4eab9ec99d35aa0c8a41, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/bedc50e1e5b440b0b14341652da48bd1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/3200bd77247947cbb43d1c99b5b3da8c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0d5d32c1aead437e88235f3549bd320d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c5069cd0846846c8bd9307c8d1e15792, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0201ce8093d5402a8debd922af29aa87, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/cb0811d09fe045df9759179ba82f621c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/7083d56efb3f420aa687cea9988b4292, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/66135e836e404f36b1526ba744254341, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/3708445ae27741b5afd99e557fdf6250, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/4bfbfdd0439946048fdb8d34e53b4c73, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/02f79751ba85471681a051ed39d9c05e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/98bfd28af1cd4991b0f2711cc5b0d01d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/822a232bfada435aae2536c95b07f6f3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/09d3cedc940444b49bb241782136f0c2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/a34e7073f70040a4b439e9f6479c5a7f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/ec00d167a93041bcbc4970d4051553c2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/18fe1a4652874193a9c323331ce92627, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/192145a293414edca02adcec58eb1cf2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/fbbd094e24f74a99acc49857fab6dcc4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/f18fe92a410e434c88d69d6fb2d2948f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/4c4055b4a3644df780ec0968710aca26, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/39b159d181ae4a51a8a8c80fedf550ed, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/ca65448ddf4746e4a4d53d9f9c2129ab, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/1e32474ce21e4d3d95f24152ee0c8d72, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/babe9f263afe4c66b1bfe88d33d244a2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/a912dbb1385e47eb8a207d405eb86ad7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0b552526819c49ad854dad8cd58f303b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/bf14cfa060514a93901b7a8cee3f280b] to archive 2024-11-16T14:15:48,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:15:48,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/515d7c98d82a4eab9ec99d35aa0c8a41 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/515d7c98d82a4eab9ec99d35aa0c8a41 2024-11-16T14:15:48,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/bedc50e1e5b440b0b14341652da48bd1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/bedc50e1e5b440b0b14341652da48bd1 2024-11-16T14:15:48,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/3200bd77247947cbb43d1c99b5b3da8c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/3200bd77247947cbb43d1c99b5b3da8c 2024-11-16T14:15:48,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0d5d32c1aead437e88235f3549bd320d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0d5d32c1aead437e88235f3549bd320d 2024-11-16T14:15:48,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c5069cd0846846c8bd9307c8d1e15792 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c5069cd0846846c8bd9307c8d1e15792 2024-11-16T14:15:48,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0201ce8093d5402a8debd922af29aa87 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0201ce8093d5402a8debd922af29aa87 2024-11-16T14:15:48,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/cb0811d09fe045df9759179ba82f621c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/cb0811d09fe045df9759179ba82f621c 2024-11-16T14:15:48,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/7083d56efb3f420aa687cea9988b4292 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/7083d56efb3f420aa687cea9988b4292 2024-11-16T14:15:48,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/66135e836e404f36b1526ba744254341 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/66135e836e404f36b1526ba744254341 2024-11-16T14:15:48,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/3708445ae27741b5afd99e557fdf6250 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/3708445ae27741b5afd99e557fdf6250 2024-11-16T14:15:48,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/4bfbfdd0439946048fdb8d34e53b4c73 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/4bfbfdd0439946048fdb8d34e53b4c73 2024-11-16T14:15:48,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/02f79751ba85471681a051ed39d9c05e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/02f79751ba85471681a051ed39d9c05e 2024-11-16T14:15:48,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/98bfd28af1cd4991b0f2711cc5b0d01d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/98bfd28af1cd4991b0f2711cc5b0d01d 2024-11-16T14:15:48,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/822a232bfada435aae2536c95b07f6f3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/822a232bfada435aae2536c95b07f6f3 2024-11-16T14:15:48,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/09d3cedc940444b49bb241782136f0c2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/09d3cedc940444b49bb241782136f0c2 2024-11-16T14:15:48,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/a34e7073f70040a4b439e9f6479c5a7f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/a34e7073f70040a4b439e9f6479c5a7f 2024-11-16T14:15:48,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/ec00d167a93041bcbc4970d4051553c2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/ec00d167a93041bcbc4970d4051553c2 2024-11-16T14:15:48,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/18fe1a4652874193a9c323331ce92627 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/18fe1a4652874193a9c323331ce92627 2024-11-16T14:15:48,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/192145a293414edca02adcec58eb1cf2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/192145a293414edca02adcec58eb1cf2 2024-11-16T14:15:48,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/fbbd094e24f74a99acc49857fab6dcc4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/fbbd094e24f74a99acc49857fab6dcc4 2024-11-16T14:15:48,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/f18fe92a410e434c88d69d6fb2d2948f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/f18fe92a410e434c88d69d6fb2d2948f 2024-11-16T14:15:48,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/4c4055b4a3644df780ec0968710aca26 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/4c4055b4a3644df780ec0968710aca26 2024-11-16T14:15:48,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/39b159d181ae4a51a8a8c80fedf550ed to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/39b159d181ae4a51a8a8c80fedf550ed 2024-11-16T14:15:48,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/ca65448ddf4746e4a4d53d9f9c2129ab to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/ca65448ddf4746e4a4d53d9f9c2129ab 2024-11-16T14:15:48,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/1e32474ce21e4d3d95f24152ee0c8d72 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/1e32474ce21e4d3d95f24152ee0c8d72 2024-11-16T14:15:48,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/babe9f263afe4c66b1bfe88d33d244a2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/babe9f263afe4c66b1bfe88d33d244a2 2024-11-16T14:15:48,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/a912dbb1385e47eb8a207d405eb86ad7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/a912dbb1385e47eb8a207d405eb86ad7 2024-11-16T14:15:48,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0b552526819c49ad854dad8cd58f303b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/0b552526819c49ad854dad8cd58f303b 2024-11-16T14:15:48,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/bf14cfa060514a93901b7a8cee3f280b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/bf14cfa060514a93901b7a8cee3f280b 2024-11-16T14:15:48,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/7adee86731284014ae3ea88bcae66148, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/11a00d6e555d47f28aeb2e3fe5d0115c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/acfdba2062944cef8bb8cbec0adf48d3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/e4914ed20f1f4b29a1a74ec9cc078bbb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/5ed31532b1bf4632bd97afd095359a03, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/5bb2fb2880934d9c8011d771fd02d1bb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/9cc3df17ac014202870c4d835d5f1c56, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/43b5955156364fed96ed1db43ed60cdd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/f4fc456462c34dd7ace38268c7f222f3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/4ca8e3179e6e4decb1a6a7640c268ead, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/683cbe533b7a4e12a8c83e38a19b43c2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/71fb32536491404886d81d2c984f8673, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/0a02b03cdfbf4fd2851ab8ddc28e0c03, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/1ac16c2a8b9546fbad3244426bf1f984, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/84cb95533db64f1ea8dd13fbf73a53e9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/523b653880684c16ada131e290cba11b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/17823036a7154c208898608b37fe08bd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/2a7b2b579daf4f27a5360934f797122d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/6296c15d265442a7a4840db17181af4e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/7b477bf1df3d4a0c869faa3e0341a557, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/f3637db4491040b88f5d734cfe8e3b41, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/ed120a7569a04b789a343ee04bb10ff3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/508965ffe52540a9a33d0269da3e4ab0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/53136f6107bd4241a2a9cd8b1f17be24, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/e301d159630e43e297f0c3ef56609cb7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/da7e56971e2a4670857ac5c680c8a057, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/ec8527fa15564ba68f7807d6196a808e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/661a884e897747cb992f43dc7d23ce47, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/671300e5bedf40e39591cdb57ed97bf0] to archive 2024-11-16T14:15:48,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:15:48,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/7adee86731284014ae3ea88bcae66148 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/7adee86731284014ae3ea88bcae66148 2024-11-16T14:15:48,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/11a00d6e555d47f28aeb2e3fe5d0115c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/11a00d6e555d47f28aeb2e3fe5d0115c 2024-11-16T14:15:48,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/acfdba2062944cef8bb8cbec0adf48d3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/acfdba2062944cef8bb8cbec0adf48d3 2024-11-16T14:15:48,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/e4914ed20f1f4b29a1a74ec9cc078bbb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/e4914ed20f1f4b29a1a74ec9cc078bbb 2024-11-16T14:15:48,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/5ed31532b1bf4632bd97afd095359a03 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/5ed31532b1bf4632bd97afd095359a03 2024-11-16T14:15:48,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/5bb2fb2880934d9c8011d771fd02d1bb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/5bb2fb2880934d9c8011d771fd02d1bb 2024-11-16T14:15:48,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/9cc3df17ac014202870c4d835d5f1c56 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/9cc3df17ac014202870c4d835d5f1c56 2024-11-16T14:15:48,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/43b5955156364fed96ed1db43ed60cdd to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/43b5955156364fed96ed1db43ed60cdd 2024-11-16T14:15:48,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/f4fc456462c34dd7ace38268c7f222f3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/f4fc456462c34dd7ace38268c7f222f3 2024-11-16T14:15:48,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/4ca8e3179e6e4decb1a6a7640c268ead to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/4ca8e3179e6e4decb1a6a7640c268ead 2024-11-16T14:15:48,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/683cbe533b7a4e12a8c83e38a19b43c2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/683cbe533b7a4e12a8c83e38a19b43c2 2024-11-16T14:15:48,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/71fb32536491404886d81d2c984f8673 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/71fb32536491404886d81d2c984f8673 2024-11-16T14:15:48,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/0a02b03cdfbf4fd2851ab8ddc28e0c03 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/0a02b03cdfbf4fd2851ab8ddc28e0c03 2024-11-16T14:15:48,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/1ac16c2a8b9546fbad3244426bf1f984 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/1ac16c2a8b9546fbad3244426bf1f984 2024-11-16T14:15:48,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/84cb95533db64f1ea8dd13fbf73a53e9 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/84cb95533db64f1ea8dd13fbf73a53e9 2024-11-16T14:15:48,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/523b653880684c16ada131e290cba11b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/523b653880684c16ada131e290cba11b 2024-11-16T14:15:48,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/17823036a7154c208898608b37fe08bd to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/17823036a7154c208898608b37fe08bd 2024-11-16T14:15:48,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/2a7b2b579daf4f27a5360934f797122d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/2a7b2b579daf4f27a5360934f797122d 2024-11-16T14:15:48,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/6296c15d265442a7a4840db17181af4e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/6296c15d265442a7a4840db17181af4e 2024-11-16T14:15:48,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/7b477bf1df3d4a0c869faa3e0341a557 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/7b477bf1df3d4a0c869faa3e0341a557 2024-11-16T14:15:48,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/f3637db4491040b88f5d734cfe8e3b41 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/f3637db4491040b88f5d734cfe8e3b41 2024-11-16T14:15:48,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/ed120a7569a04b789a343ee04bb10ff3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/ed120a7569a04b789a343ee04bb10ff3 2024-11-16T14:15:48,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/508965ffe52540a9a33d0269da3e4ab0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/508965ffe52540a9a33d0269da3e4ab0 2024-11-16T14:15:48,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/53136f6107bd4241a2a9cd8b1f17be24 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/53136f6107bd4241a2a9cd8b1f17be24 2024-11-16T14:15:48,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/e301d159630e43e297f0c3ef56609cb7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/e301d159630e43e297f0c3ef56609cb7 2024-11-16T14:15:48,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/da7e56971e2a4670857ac5c680c8a057 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/da7e56971e2a4670857ac5c680c8a057 2024-11-16T14:15:48,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/ec8527fa15564ba68f7807d6196a808e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/ec8527fa15564ba68f7807d6196a808e 2024-11-16T14:15:48,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/661a884e897747cb992f43dc7d23ce47 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/661a884e897747cb992f43dc7d23ce47 2024-11-16T14:15:48,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/671300e5bedf40e39591cdb57ed97bf0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/671300e5bedf40e39591cdb57ed97bf0 2024-11-16T14:15:48,477 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/recovered.edits/693.seqid, newMaxSeqId=693, maxSeqId=1 2024-11-16T14:15:48,477 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. 2024-11-16T14:15:48,477 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 5bda05ae08ffa563beaf32369a8bf2d2: Waiting for close lock at 1731766547116Running coprocessor pre-close hooks at 1731766547116Disabling compacts and flushes for region at 1731766547116Disabling writes for close at 1731766547117 (+1 ms)Obtaining lock to block concurrent updates at 1731766547117Preparing flush snapshotting stores in 5bda05ae08ffa563beaf32369a8bf2d2 at 1731766547117Finished memstore snapshotting TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731766547118 (+1 ms)Flushing stores of TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2. at 1731766547123 (+5 ms)Flushing 5bda05ae08ffa563beaf32369a8bf2d2/A: creating writer at 1731766547123Flushing 5bda05ae08ffa563beaf32369a8bf2d2/A: appending metadata at 1731766547131 (+8 ms)Flushing 5bda05ae08ffa563beaf32369a8bf2d2/A: closing flushed file at 1731766547131Flushing 5bda05ae08ffa563beaf32369a8bf2d2/B: creating writer at 1731766547555 (+424 ms)Flushing 5bda05ae08ffa563beaf32369a8bf2d2/B: appending metadata at 1731766547560 (+5 ms)Flushing 5bda05ae08ffa563beaf32369a8bf2d2/B: closing flushed file at 1731766547560Flushing 5bda05ae08ffa563beaf32369a8bf2d2/C: creating writer at 1731766547973 (+413 ms)Flushing 5bda05ae08ffa563beaf32369a8bf2d2/C: appending metadata at 1731766547977 (+4 ms)Flushing 5bda05ae08ffa563beaf32369a8bf2d2/C: closing flushed file at 1731766547977Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ee90877: reopening flushed file at 1731766548385 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59ac26aa: reopening flushed file at 1731766548392 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77bfc6bd: reopening flushed file at 1731766548397 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5bda05ae08ffa563beaf32369a8bf2d2 in 1285ms, sequenceid=690, compaction requested=true at 1731766548402 (+5 ms)Writing region close event to WAL at 1731766548474 (+72 ms)Running coprocessor post-close hooks at 1731766548477 (+3 ms)Closed at 1731766548477 2024-11-16T14:15:48,479 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:48,479 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5bda05ae08ffa563beaf32369a8bf2d2, regionState=CLOSED 2024-11-16T14:15:48,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:15:48,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-16T14:15:48,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 5bda05ae08ffa563beaf32369a8bf2d2, server=04e93f2b5fad,41317,1731766448342 in 1.5180 sec 2024-11-16T14:15:48,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-16T14:15:48,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5bda05ae08ffa563beaf32369a8bf2d2, UNASSIGN in 1.5230 sec 2024-11-16T14:15:48,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-16T14:15:48,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5280 sec 2024-11-16T14:15:48,486 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766548486"}]},"ts":"1731766548486"} 2024-11-16T14:15:48,487 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T14:15:48,487 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T14:15:48,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5380 sec 2024-11-16T14:15:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T14:15:49,096 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:49,097 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T14:15:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:49,098 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T14:15:49,099 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:49,102 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:49,104 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/recovered.edits] 2024-11-16T14:15:49,106 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/79549a23a3784b53b53374b820ba2ea5 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/79549a23a3784b53b53374b820ba2ea5 2024-11-16T14:15:49,107 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8ebb0b0cc3864043a5b237874f9c7c00 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/8ebb0b0cc3864043a5b237874f9c7c00 2024-11-16T14:15:49,108 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/a0d8f28e60f941c387b07761a1693812 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/a0d8f28e60f941c387b07761a1693812 2024-11-16T14:15:49,109 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/b8f3790777bb4ed0b941eaf454f3ac64 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/A/b8f3790777bb4ed0b941eaf454f3ac64 2024-11-16T14:15:49,111 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/579538999c944a2c99a5e8b99c126a45 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/579538999c944a2c99a5e8b99c126a45 2024-11-16T14:15:49,112 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/8c701313edd147d895d291aaed8a83ea to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/8c701313edd147d895d291aaed8a83ea 2024-11-16T14:15:49,113 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c1bbac8a40de4bb8bc47ce6cb8a54e73 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c1bbac8a40de4bb8bc47ce6cb8a54e73 2024-11-16T14:15:49,114 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c3dac83154364dc68df67bf79d746e9d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/B/c3dac83154364dc68df67bf79d746e9d 2024-11-16T14:15:49,116 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/05826e62b33142f9b040337d4d8a3d83 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/05826e62b33142f9b040337d4d8a3d83 2024-11-16T14:15:49,117 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/2c2b455843b44d0dbae18588e2888ebc to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/2c2b455843b44d0dbae18588e2888ebc 2024-11-16T14:15:49,118 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/a1c5597797564f22b7b9a5cde5c1f106 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/a1c5597797564f22b7b9a5cde5c1f106 2024-11-16T14:15:49,119 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/bb83c17b94fc45d08d589d0a00659a4b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/C/bb83c17b94fc45d08d589d0a00659a4b 2024-11-16T14:15:49,121 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/recovered.edits/693.seqid to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2/recovered.edits/693.seqid 2024-11-16T14:15:49,122 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/5bda05ae08ffa563beaf32369a8bf2d2 2024-11-16T14:15:49,122 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T14:15:49,123 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:49,126 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T14:15:49,128 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T14:15:49,129 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:49,129 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T14:15:49,129 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731766549129"}]},"ts":"9223372036854775807"} 2024-11-16T14:15:49,131 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T14:15:49,131 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5bda05ae08ffa563beaf32369a8bf2d2, NAME => 'TestAcidGuarantees,,1731766521905.5bda05ae08ffa563beaf32369a8bf2d2.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T14:15:49,131 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T14:15:49,132 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731766549131"}]},"ts":"9223372036854775807"} 2024-11-16T14:15:49,134 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T14:15:49,135 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:49,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-16T14:15:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T14:15:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T14:15:49,206 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:49,216 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-571885164_22 at /127.0.0.1:57966 [Waiting for operation #244] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1474060206_22 at /127.0.0.1:50240 [Waiting for operation #361] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 374) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4018 (was 4782) 2024-11-16T14:15:49,227 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=438, ProcessCount=11, AvailableMemoryMB=4018 2024-11-16T14:15:49,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:15:49,229 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:15:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:49,231 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:15:49,231 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:49,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-16T14:15:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T14:15:49,232 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:15:49,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742158_1334 (size=1061) 2024-11-16T14:15:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T14:15:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T14:15:49,639 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc 2024-11-16T14:15:49,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742159_1335 (size=53) 2024-11-16T14:15:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T14:15:50,046 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:15:50,046 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f43c903d1ca08ce04b9ee6d4b3aa8643, disabling compactions & flushes 2024-11-16T14:15:50,046 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:50,046 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:50,047 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. after waiting 0 ms 2024-11-16T14:15:50,047 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:50,047 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:50,047 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f43c903d1ca08ce04b9ee6d4b3aa8643: Waiting for close lock at 1731766550046Disabling compacts and flushes for region at 1731766550046Disabling writes for close at 1731766550047 (+1 ms)Writing region close event to WAL at 1731766550047Closed at 1731766550047 2024-11-16T14:15:50,049 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:15:50,050 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731766550049"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731766550049"}]},"ts":"1731766550049"} 2024-11-16T14:15:50,053 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:15:50,055 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:15:50,055 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766550055"}]},"ts":"1731766550055"} 2024-11-16T14:15:50,057 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T14:15:50,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, ASSIGN}] 2024-11-16T14:15:50,058 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, ASSIGN 2024-11-16T14:15:50,059 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, ASSIGN; state=OFFLINE, location=04e93f2b5fad,41317,1731766448342; forceNewPlan=false, retain=false 2024-11-16T14:15:50,210 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=f43c903d1ca08ce04b9ee6d4b3aa8643, regionState=OPENING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:50,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, ASSIGN because future has completed 2024-11-16T14:15:50,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:15:50,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T14:15:50,369 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:50,369 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:15:50,370 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,370 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:15:50,370 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,370 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,371 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,372 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:15:50,372 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f43c903d1ca08ce04b9ee6d4b3aa8643 columnFamilyName A 2024-11-16T14:15:50,372 DEBUG [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:50,373 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.HStore(327): Store=f43c903d1ca08ce04b9ee6d4b3aa8643/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:15:50,373 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,374 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:15:50,374 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f43c903d1ca08ce04b9ee6d4b3aa8643 columnFamilyName B 2024-11-16T14:15:50,374 DEBUG [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:50,374 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.HStore(327): Store=f43c903d1ca08ce04b9ee6d4b3aa8643/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:15:50,374 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,375 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:15:50,375 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f43c903d1ca08ce04b9ee6d4b3aa8643 columnFamilyName C 2024-11-16T14:15:50,375 DEBUG [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:50,375 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.HStore(327): Store=f43c903d1ca08ce04b9ee6d4b3aa8643/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:15:50,376 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:50,376 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,376 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,377 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,378 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,378 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,378 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:15:50,379 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,381 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:15:50,381 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened f43c903d1ca08ce04b9ee6d4b3aa8643; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69503380, jitterRate=0.035681068897247314}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:15:50,381 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:50,382 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for f43c903d1ca08ce04b9ee6d4b3aa8643: Running coprocessor pre-open hook at 1731766550370Writing region info on filesystem at 1731766550370Initializing all the Stores at 1731766550371 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766550371Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766550371Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766550371Cleaning up temporary data from old regions at 1731766550378 (+7 ms)Running coprocessor post-open hooks at 1731766550381 (+3 ms)Region opened successfully at 1731766550382 (+1 ms) 2024-11-16T14:15:50,382 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., pid=89, masterSystemTime=1731766550367 2024-11-16T14:15:50,384 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:50,384 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:50,384 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=f43c903d1ca08ce04b9ee6d4b3aa8643, regionState=OPEN, openSeqNum=2, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:50,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:15:50,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-16T14:15:50,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 in 173 msec 2024-11-16T14:15:50,389 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-16T14:15:50,389 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, ASSIGN in 330 msec 2024-11-16T14:15:50,389 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:15:50,390 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766550389"}]},"ts":"1731766550389"} 2024-11-16T14:15:50,391 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T14:15:50,391 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:15:50,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-16T14:15:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T14:15:51,386 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:51,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:51,387 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:51,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:51,388 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428464f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:51,390 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:51,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3918a791, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:51,391 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:51,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:51,394 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:15:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 04e93f2b5fad,39231,1731766447537 2024-11-16T14:15:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@291a204e 2024-11-16T14:15:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:15:51,396 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:15:51,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:15:51,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:15:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T14:15:51,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742160_1336 (size=1097) 2024-11-16T14:15:51,813 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T14:15:51,813 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T14:15:51,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:15:51,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, REOPEN/MOVE}] 2024-11-16T14:15:51,823 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, REOPEN/MOVE 2024-11-16T14:15:51,824 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=f43c903d1ca08ce04b9ee6d4b3aa8643, regionState=CLOSING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:51,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, REOPEN/MOVE because future has completed 2024-11-16T14:15:51,826 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:15:51,826 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:15:51,979 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:51,979 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:15:51,979 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing f43c903d1ca08ce04b9ee6d4b3aa8643, disabling compactions & flushes 2024-11-16T14:15:51,979 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:51,979 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:51,980 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. after waiting 0 ms 2024-11-16T14:15:51,980 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:51,985 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T14:15:51,985 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:51,986 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for f43c903d1ca08ce04b9ee6d4b3aa8643: Waiting for close lock at 1731766551979Running coprocessor pre-close hooks at 1731766551979Disabling compacts and flushes for region at 1731766551979Disabling writes for close at 1731766551980 (+1 ms)Writing region close event to WAL at 1731766551980Running coprocessor post-close hooks at 1731766551985 (+5 ms)Closed at 1731766551985 2024-11-16T14:15:51,986 WARN [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: f43c903d1ca08ce04b9ee6d4b3aa8643 to self. 2024-11-16T14:15:51,987 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:51,988 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=f43c903d1ca08ce04b9ee6d4b3aa8643, regionState=CLOSED 2024-11-16T14:15:51,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:15:51,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-16T14:15:51,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 in 164 msec 2024-11-16T14:15:51,992 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, REOPEN/MOVE; state=CLOSED, location=04e93f2b5fad,41317,1731766448342; forceNewPlan=false, retain=true 2024-11-16T14:15:52,143 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=f43c903d1ca08ce04b9ee6d4b3aa8643, regionState=OPENING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, REOPEN/MOVE because future has completed 2024-11-16T14:15:52,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:15:52,302 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:52,302 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:15:52,302 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,302 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:15:52,302 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,302 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,303 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,304 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:15:52,304 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f43c903d1ca08ce04b9ee6d4b3aa8643 columnFamilyName A 2024-11-16T14:15:52,305 DEBUG [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:52,306 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.HStore(327): Store=f43c903d1ca08ce04b9ee6d4b3aa8643/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:15:52,306 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,307 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:15:52,307 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f43c903d1ca08ce04b9ee6d4b3aa8643 columnFamilyName B 2024-11-16T14:15:52,307 DEBUG [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:52,307 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.HStore(327): Store=f43c903d1ca08ce04b9ee6d4b3aa8643/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:15:52,307 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,307 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:15:52,308 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f43c903d1ca08ce04b9ee6d4b3aa8643 columnFamilyName C 2024-11-16T14:15:52,308 DEBUG [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:52,308 INFO [StoreOpener-f43c903d1ca08ce04b9ee6d4b3aa8643-1 {}] regionserver.HStore(327): Store=f43c903d1ca08ce04b9ee6d4b3aa8643/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:15:52,308 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:52,308 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,308 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,309 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,310 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,310 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,310 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:15:52,312 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,312 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened f43c903d1ca08ce04b9ee6d4b3aa8643; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62467242, jitterRate=-0.06916555762290955}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:15:52,312 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,313 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for f43c903d1ca08ce04b9ee6d4b3aa8643: Running coprocessor pre-open hook at 1731766552302Writing region info on filesystem at 1731766552302Initializing all the Stores at 1731766552303 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766552303Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766552303Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766552303Cleaning up temporary data from old regions at 1731766552310 (+7 ms)Running coprocessor post-open hooks at 1731766552312 (+2 ms)Region opened successfully at 1731766552313 (+1 ms) 2024-11-16T14:15:52,314 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., pid=94, masterSystemTime=1731766552299 2024-11-16T14:15:52,315 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:52,315 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:52,316 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=f43c903d1ca08ce04b9ee6d4b3aa8643, regionState=OPEN, openSeqNum=5, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:15:52,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-16T14:15:52,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 in 171 msec 2024-11-16T14:15:52,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-16T14:15:52,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, REOPEN/MOVE in 498 msec 2024-11-16T14:15:52,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-16T14:15:52,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-11-16T14:15:52,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 924 msec 2024-11-16T14:15:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-16T14:15:52,325 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:52,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:52,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:52,327 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd929ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:52,328 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c257831, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:52,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:52,330 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3b3837, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:52,331 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42dd8c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:52,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:52,333 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:52,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:52,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f55d80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:52,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:52,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:52,335 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@214616c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:52,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:52,337 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c4fac19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:52,339 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4379c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:52,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:52,341 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@de9b5cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:52,343 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@94b1ac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:52,344 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:52,345 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bfbb208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:52,346 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5ca4ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:52,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:52,348 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108659b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:52,349 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:52,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42833065, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:52,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:52,351 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dce65c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:52,353 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:52,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@671a8f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,354 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:52,354 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:52,355 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ef8181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:52,356 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dce3f46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:15:52,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:15:52,358 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:15:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:15:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:15:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ef9dc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:15:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:15:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:15:52,359 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:15:52,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e23c30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:15:52,361 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:52,361 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:52,361 DEBUG [Thread-1543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:52,361 DEBUG [Thread-1545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:52,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:52,363 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:52,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:52,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:52,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:52,364 DEBUG [Thread-1548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:52,364 DEBUG [Thread-1552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:52,365 DEBUG [Thread-1552 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:52,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:52,365 DEBUG [Thread-1552 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:15:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:15:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:15:52,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:52,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:52,367 DEBUG [Thread-1550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:52,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-16T14:15:52,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:15:52,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:15:52,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:52,371 DEBUG [Thread-1541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:15:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:15:52,371 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:52,372 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:52,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:52,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:52,374 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:52,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:15:52,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:52,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:15:52,381 DEBUG [Thread-1537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:52,381 DEBUG [Thread-1539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:15:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:15:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:15:52,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:52,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:15:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:15:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:15:52,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:15:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:15:52,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:52,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:15:52,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:52,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:15:52,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:52,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161ed1907f8e9247f8a0a790fd07fd32d1_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766552370/Put/seqid=0 2024-11-16T14:15:52,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766552409, max timestamp=1731766552452Num uniques 859; 2024-11-16T14:15:52,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:52,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766552409, max timestamp=1731766552452Num uniques 859; 2024-11-16T14:15:52,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:52,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766552409, max timestamp=1731766552452Num uniques 859; 2024-11-16T14:15:52,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:15:52,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742161_1337 (size=12154) 2024-11-16T14:15:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:15:52,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766612484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766612487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766612488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766612489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766612497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:15:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:52,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:52,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766612606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766612608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766612616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766612616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766612616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:15:52,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:52,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:52,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:52,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:52,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:15:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766612816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766612816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:15:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:52,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:52,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766612836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766612836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766612836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:52,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:52,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:52,873 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:52,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161ed1907f8e9247f8a0a790fd07fd32d1_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161ed1907f8e9247f8a0a790fd07fd32d1_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:52,880 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/795aacd380b341e1b43fe6d0894befcb, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:15:52,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/795aacd380b341e1b43fe6d0894befcb is 175, key is test_row_0/A:col10/1731766552370/Put/seqid=0 2024-11-16T14:15:52,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742162_1338 (size=30955) 2024-11-16T14:15:52,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:15:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:52,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:15:53,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766613136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:53,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766613136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:53,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:15:53,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:53,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:53,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766613146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:53,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:53,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766613146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:53,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:53,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766613156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:53,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:15:53,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:53,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:53,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:53,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,309 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/795aacd380b341e1b43fe6d0894befcb 2024-11-16T14:15:53,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/8ab0ce8dfacd48ebb0857bf17671db69 is 50, key is test_row_0/B:col10/1731766552370/Put/seqid=0 2024-11-16T14:15:53,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742163_1339 (size=12001) 2024-11-16T14:15:53,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:15:53,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:53,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:53,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:53,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:15:53,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:15:53,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:53,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:53,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:53,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:53,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766613656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:53,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766613656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:53,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766613656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:53,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766613665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:53,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766613675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:53,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/8ab0ce8dfacd48ebb0857bf17671db69 2024-11-16T14:15:53,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/6a046fe006654047b8be25661a395073 is 50, key is test_row_0/C:col10/1731766552370/Put/seqid=0 2024-11-16T14:15:53,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742164_1340 (size=12001) 2024-11-16T14:15:53,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/6a046fe006654047b8be25661a395073 2024-11-16T14:15:53,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/795aacd380b341e1b43fe6d0894befcb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/795aacd380b341e1b43fe6d0894befcb 2024-11-16T14:15:53,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/795aacd380b341e1b43fe6d0894befcb, entries=150, sequenceid=16, filesize=30.2 K 2024-11-16T14:15:53,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:15:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:53,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:53,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/8ab0ce8dfacd48ebb0857bf17671db69 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/8ab0ce8dfacd48ebb0857bf17671db69 2024-11-16T14:15:53,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/8ab0ce8dfacd48ebb0857bf17671db69, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T14:15:53,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/6a046fe006654047b8be25661a395073 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/6a046fe006654047b8be25661a395073 2024-11-16T14:15:53,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/6a046fe006654047b8be25661a395073, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T14:15:53,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1430ms, sequenceid=16, compaction requested=false 2024-11-16T14:15:53,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:15:53,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:15:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:53,960 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-16T14:15:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:15:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:15:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:15:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:15:53,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164345b757e7824f87a15dd96bc9eeaaf5_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766552486/Put/seqid=0 2024-11-16T14:15:53,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742165_1341 (size=12154) 2024-11-16T14:15:54,006 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:15:54,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:54,378 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164345b757e7824f87a15dd96bc9eeaaf5_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164345b757e7824f87a15dd96bc9eeaaf5_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:54,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/b46193a2e4514fa28e6397f7a1344ed9, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:15:54,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/b46193a2e4514fa28e6397f7a1344ed9 is 175, key is test_row_0/A:col10/1731766552486/Put/seqid=0 2024-11-16T14:15:54,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742166_1342 (size=30955) 2024-11-16T14:15:54,400 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/b46193a2e4514fa28e6397f7a1344ed9 2024-11-16T14:15:54,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/0251b3e1b26d4cb7887b3795859c3e31 is 50, key is test_row_0/B:col10/1731766552486/Put/seqid=0 2024-11-16T14:15:54,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742167_1343 (size=12001) 2024-11-16T14:15:54,425 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/0251b3e1b26d4cb7887b3795859c3e31 2024-11-16T14:15:54,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/02d3c7f7d0bc4df59cfcd1fec042cd8c is 50, key is test_row_0/C:col10/1731766552486/Put/seqid=0 2024-11-16T14:15:54,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742168_1344 (size=12001) 2024-11-16T14:15:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:15:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:54,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766614696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:54,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:54,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766614697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766614698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766614700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:54,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766614701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:54,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766614805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:54,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766614815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766614815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:54,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766614816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:54,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766614826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:54,846 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/02d3c7f7d0bc4df59cfcd1fec042cd8c 2024-11-16T14:15:54,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/b46193a2e4514fa28e6397f7a1344ed9 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b46193a2e4514fa28e6397f7a1344ed9 2024-11-16T14:15:54,854 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b46193a2e4514fa28e6397f7a1344ed9, entries=150, sequenceid=50, filesize=30.2 K 2024-11-16T14:15:54,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/0251b3e1b26d4cb7887b3795859c3e31 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/0251b3e1b26d4cb7887b3795859c3e31 2024-11-16T14:15:54,858 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/0251b3e1b26d4cb7887b3795859c3e31, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T14:15:54,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/02d3c7f7d0bc4df59cfcd1fec042cd8c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/02d3c7f7d0bc4df59cfcd1fec042cd8c 2024-11-16T14:15:54,863 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/02d3c7f7d0bc4df59cfcd1fec042cd8c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T14:15:54,864 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 903ms, sequenceid=50, compaction requested=false 2024-11-16T14:15:54,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:15:54,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:54,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-16T14:15:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-16T14:15:54,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-16T14:15:54,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4950 sec 2024-11-16T14:15:54,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.5080 sec 2024-11-16T14:15:55,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:15:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:15:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:15:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:15:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:55,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160179adb37a1e41f4a20a64d1d9c68bb9_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766555029/Put/seqid=0 2024-11-16T14:15:55,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742169_1345 (size=14594) 2024-11-16T14:15:55,058 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:55,068 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160179adb37a1e41f4a20a64d1d9c68bb9_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160179adb37a1e41f4a20a64d1d9c68bb9_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:55,069 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/1cd320732ff446b698e34fc499914896, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:15:55,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/1cd320732ff446b698e34fc499914896 is 175, key is test_row_0/A:col10/1731766555029/Put/seqid=0 2024-11-16T14:15:55,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766615079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766615080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766615084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766615086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742170_1346 (size=39549) 2024-11-16T14:15:55,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766615088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,107 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/1cd320732ff446b698e34fc499914896 2024-11-16T14:15:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/893fdbddecda40b6a2965c0fbb852a57 is 50, key is test_row_0/B:col10/1731766555029/Put/seqid=0 2024-11-16T14:15:55,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742171_1347 (size=12001) 2024-11-16T14:15:55,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/893fdbddecda40b6a2965c0fbb852a57 2024-11-16T14:15:55,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/8ef0fb9d4fdc4d638c537d003c845bc4 is 50, key is test_row_0/C:col10/1731766555029/Put/seqid=0 2024-11-16T14:15:55,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742172_1348 (size=12001) 2024-11-16T14:15:55,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/8ef0fb9d4fdc4d638c537d003c845bc4 2024-11-16T14:15:55,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/1cd320732ff446b698e34fc499914896 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1cd320732ff446b698e34fc499914896 2024-11-16T14:15:55,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1cd320732ff446b698e34fc499914896, entries=200, sequenceid=63, filesize=38.6 K 2024-11-16T14:15:55,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/893fdbddecda40b6a2965c0fbb852a57 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/893fdbddecda40b6a2965c0fbb852a57 2024-11-16T14:15:55,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/893fdbddecda40b6a2965c0fbb852a57, entries=150, sequenceid=63, filesize=11.7 K 2024-11-16T14:15:55,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/8ef0fb9d4fdc4d638c537d003c845bc4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/8ef0fb9d4fdc4d638c537d003c845bc4 2024-11-16T14:15:55,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766615195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766615206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766615206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766615206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/8ef0fb9d4fdc4d638c537d003c845bc4, entries=150, sequenceid=63, filesize=11.7 K 2024-11-16T14:15:55,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 182ms, sequenceid=63, compaction requested=true 2024-11-16T14:15:55,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:15:55,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:15:55,213 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:55,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:55,213 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:55,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:15:55,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:55,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:15:55,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:55,214 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:55,214 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/B is initiating minor compaction (all files) 2024-11-16T14:15:55,214 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/B in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:55,214 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/8ab0ce8dfacd48ebb0857bf17671db69, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/0251b3e1b26d4cb7887b3795859c3e31, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/893fdbddecda40b6a2965c0fbb852a57] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=35.2 K 2024-11-16T14:15:55,214 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:55,214 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/A is initiating minor compaction (all files) 2024-11-16T14:15:55,214 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/A in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:55,215 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/795aacd380b341e1b43fe6d0894befcb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b46193a2e4514fa28e6397f7a1344ed9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1cd320732ff446b698e34fc499914896] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=99.1 K 2024-11-16T14:15:55,215 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ab0ce8dfacd48ebb0857bf17671db69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731766552370 2024-11-16T14:15:55,215 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:55,215 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/795aacd380b341e1b43fe6d0894befcb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b46193a2e4514fa28e6397f7a1344ed9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1cd320732ff446b698e34fc499914896] 2024-11-16T14:15:55,215 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0251b3e1b26d4cb7887b3795859c3e31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731766552485 2024-11-16T14:15:55,215 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 795aacd380b341e1b43fe6d0894befcb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731766552370 2024-11-16T14:15:55,216 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b46193a2e4514fa28e6397f7a1344ed9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731766552485 2024-11-16T14:15:55,216 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 893fdbddecda40b6a2965c0fbb852a57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731766554696 2024-11-16T14:15:55,216 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cd320732ff446b698e34fc499914896, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731766554696 2024-11-16T14:15:55,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:15:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:55,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:15:55,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:55,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:15:55,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:55,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:15:55,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:55,229 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:15:55,239 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#B#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:55,239 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/0901b2c119c94ee19a80555de1f464e3 is 50, key is test_row_0/B:col10/1731766555029/Put/seqid=0 2024-11-16T14:15:55,253 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116a861f804184c45da88b57dc797bfae64_f43c903d1ca08ce04b9ee6d4b3aa8643 store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:15:55,255 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116a861f804184c45da88b57dc797bfae64_f43c903d1ca08ce04b9ee6d4b3aa8643, store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:15:55,255 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a861f804184c45da88b57dc797bfae64_f43c903d1ca08ce04b9ee6d4b3aa8643 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:15:55,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111636cedc85b1df4ddda1e028e5ce1d97ef_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766555086/Put/seqid=0 2024-11-16T14:15:55,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742173_1349 (size=12104) 2024-11-16T14:15:55,283 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/0901b2c119c94ee19a80555de1f464e3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/0901b2c119c94ee19a80555de1f464e3 2024-11-16T14:15:55,289 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/B of f43c903d1ca08ce04b9ee6d4b3aa8643 into 0901b2c119c94ee19a80555de1f464e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:55,289 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:15:55,289 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/B, priority=13, startTime=1731766555213; duration=0sec 2024-11-16T14:15:55,289 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:15:55,289 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:B 2024-11-16T14:15:55,289 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:15:55,290 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:15:55,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742174_1350 (size=4469) 2024-11-16T14:15:55,293 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/C is initiating minor compaction (all files) 2024-11-16T14:15:55,293 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/C in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:55,294 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/6a046fe006654047b8be25661a395073, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/02d3c7f7d0bc4df59cfcd1fec042cd8c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/8ef0fb9d4fdc4d638c537d003c845bc4] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=35.2 K 2024-11-16T14:15:55,294 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a046fe006654047b8be25661a395073, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731766552370 2024-11-16T14:15:55,294 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 02d3c7f7d0bc4df59cfcd1fec042cd8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731766552485 2024-11-16T14:15:55,295 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ef0fb9d4fdc4d638c537d003c845bc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731766554696 2024-11-16T14:15:55,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742175_1351 (size=14594) 2024-11-16T14:15:55,307 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:55,307 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/f93274cec41c4079b5dc8564a6f0c161 is 50, key is test_row_0/C:col10/1731766555029/Put/seqid=0 2024-11-16T14:15:55,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742176_1352 (size=12104) 2024-11-16T14:15:55,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766615327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766615415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766615415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766615416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766615416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766615436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766615645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,695 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#A#compaction#300 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:15:55,695 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/b843b895cb9e45b4a9befd3f46fd83fe is 175, key is test_row_0/A:col10/1731766555029/Put/seqid=0 2024-11-16T14:15:55,707 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:55,710 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111636cedc85b1df4ddda1e028e5ce1d97ef_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636cedc85b1df4ddda1e028e5ce1d97ef_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:55,711 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/5132bed147c846cd82c32e06ca14a374, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:15:55,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/5132bed147c846cd82c32e06ca14a374 is 175, key is test_row_0/A:col10/1731766555086/Put/seqid=0 2024-11-16T14:15:55,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742177_1353 (size=31058) 2024-11-16T14:15:55,723 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/f93274cec41c4079b5dc8564a6f0c161 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f93274cec41c4079b5dc8564a6f0c161 2024-11-16T14:15:55,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742178_1354 (size=39549) 2024-11-16T14:15:55,727 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/C of f43c903d1ca08ce04b9ee6d4b3aa8643 into f93274cec41c4079b5dc8564a6f0c161(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:55,728 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:15:55,728 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/C, priority=13, startTime=1731766555213; duration=0sec 2024-11-16T14:15:55,728 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:55,728 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:C 2024-11-16T14:15:55,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766615735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766615735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766615736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766615736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:55,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766615965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:56,117 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/b843b895cb9e45b4a9befd3f46fd83fe as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b843b895cb9e45b4a9befd3f46fd83fe 2024-11-16T14:15:56,123 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/A of f43c903d1ca08ce04b9ee6d4b3aa8643 into b843b895cb9e45b4a9befd3f46fd83fe(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:15:56,123 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:15:56,123 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/A, priority=13, startTime=1731766555213; duration=0sec 2024-11-16T14:15:56,123 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:15:56,123 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:A 2024-11-16T14:15:56,125 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/5132bed147c846cd82c32e06ca14a374 2024-11-16T14:15:56,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/14f3c0cdaf0d4a4185fe8b7caea280d2 is 50, key is test_row_0/B:col10/1731766555086/Put/seqid=0 2024-11-16T14:15:56,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742179_1355 (size=12001) 2024-11-16T14:15:56,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766616255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:56,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766616256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:56,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766616256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:56,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766616256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:56,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:56,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:56,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:56,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766616476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:56,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/14f3c0cdaf0d4a4185fe8b7caea280d2 2024-11-16T14:15:56,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/f7eff9b39a494b5ca6ecdd9eda6f22ba is 50, key is test_row_0/C:col10/1731766555086/Put/seqid=0 2024-11-16T14:15:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:15:56,546 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:56,546 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-16T14:15:56,548 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T14:15:56,552 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:56,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:56,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742180_1356 (size=12001) 2024-11-16T14:15:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T14:15:56,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T14:15:56,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:56,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:56,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:56,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:56,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:56,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T14:15:56,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:56,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:56,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:56,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:56,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T14:15:56,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/f7eff9b39a494b5ca6ecdd9eda6f22ba 2024-11-16T14:15:56,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/5132bed147c846cd82c32e06ca14a374 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5132bed147c846cd82c32e06ca14a374 2024-11-16T14:15:56,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5132bed147c846cd82c32e06ca14a374, entries=200, sequenceid=83, filesize=38.6 K 2024-11-16T14:15:56,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/14f3c0cdaf0d4a4185fe8b7caea280d2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/14f3c0cdaf0d4a4185fe8b7caea280d2 2024-11-16T14:15:57,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/14f3c0cdaf0d4a4185fe8b7caea280d2, entries=150, sequenceid=83, filesize=11.7 K 2024-11-16T14:15:57,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/f7eff9b39a494b5ca6ecdd9eda6f22ba as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f7eff9b39a494b5ca6ecdd9eda6f22ba 2024-11-16T14:15:57,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f7eff9b39a494b5ca6ecdd9eda6f22ba, entries=150, sequenceid=83, filesize=11.7 K 2024-11-16T14:15:57,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1788ms, sequenceid=83, compaction requested=false 2024-11-16T14:15:57,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:15:57,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T14:15:57,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:57,016 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:15:57,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:15:57,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:57,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:15:57,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:57,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:15:57,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116efaf63df02c24f0bb74a0b2ee35140e1_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766555299/Put/seqid=0 2024-11-16T14:15:57,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742181_1357 (size=12154) 2024-11-16T14:15:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T14:15:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:57,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766617298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766617299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766617309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766617309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766617415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766617416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766617416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766617416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:57,436 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116efaf63df02c24f0bb74a0b2ee35140e1_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116efaf63df02c24f0bb74a0b2ee35140e1_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:57,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/27e717b3d41a45cb996891eebda1d078, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:15:57,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/27e717b3d41a45cb996891eebda1d078 is 175, key is test_row_0/A:col10/1731766555299/Put/seqid=0 2024-11-16T14:15:57,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742182_1358 (size=30955) 2024-11-16T14:15:57,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766617486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,492 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in f43c903d1ca08ce04b9ee6d4b3aa8643 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:15:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766617636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766617636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766617636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766617636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T14:15:57,850 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/27e717b3d41a45cb996891eebda1d078 2024-11-16T14:15:57,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/ea0d1ef8fabc4fd58a39af376083f434 is 50, key is test_row_0/B:col10/1731766555299/Put/seqid=0 2024-11-16T14:15:57,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742183_1359 (size=12001) 2024-11-16T14:15:57,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766617945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766617946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766617956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:57,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766617956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:58,259 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/ea0d1ef8fabc4fd58a39af376083f434 2024-11-16T14:15:58,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/1c714d99a2bd49ac871f3202a1718302 is 50, key is test_row_0/C:col10/1731766555299/Put/seqid=0 2024-11-16T14:15:58,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742184_1360 (size=12001) 2024-11-16T14:15:58,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766618466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:58,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766618466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:58,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766618475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:58,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766618475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:58,669 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/1c714d99a2bd49ac871f3202a1718302 2024-11-16T14:15:58,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/27e717b3d41a45cb996891eebda1d078 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/27e717b3d41a45cb996891eebda1d078 2024-11-16T14:15:58,677 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/27e717b3d41a45cb996891eebda1d078, entries=150, sequenceid=98, filesize=30.2 K 2024-11-16T14:15:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/ea0d1ef8fabc4fd58a39af376083f434 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ea0d1ef8fabc4fd58a39af376083f434 2024-11-16T14:15:58,683 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ea0d1ef8fabc4fd58a39af376083f434, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T14:15:58,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/1c714d99a2bd49ac871f3202a1718302 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/1c714d99a2bd49ac871f3202a1718302 2024-11-16T14:15:58,686 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/1c714d99a2bd49ac871f3202a1718302, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T14:15:58,687 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1671ms, sequenceid=98, compaction requested=true 2024-11-16T14:15:58,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:15:58,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:58,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-16T14:15:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-16T14:15:58,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-16T14:15:58,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1360 sec 2024-11-16T14:15:58,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.1460 sec 2024-11-16T14:15:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T14:15:58,696 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:15:58,696 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:15:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-16T14:15:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T14:15:58,698 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:15:58,698 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:15:58,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:15:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T14:15:58,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T14:15:58,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:15:58,849 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:15:58,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:15:58,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:58,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:15:58,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:58,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:15:58,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:15:58,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116692c4908646a476ab44bdc5c30cba260_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766557307/Put/seqid=0 2024-11-16T14:15:58,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742185_1361 (size=12154) 2024-11-16T14:15:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T14:15:59,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:15:59,268 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116692c4908646a476ab44bdc5c30cba260_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116692c4908646a476ab44bdc5c30cba260_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:59,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/bbf22413c56d47ab94ec868ce5633dd8, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:15:59,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/bbf22413c56d47ab94ec868ce5633dd8 is 175, key is test_row_0/A:col10/1731766557307/Put/seqid=0 2024-11-16T14:15:59,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742186_1362 (size=30955) 2024-11-16T14:15:59,278 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/bbf22413c56d47ab94ec868ce5633dd8 2024-11-16T14:15:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/5bd852b378aa4fb1b379acb3dab11b2b is 50, key is test_row_0/B:col10/1731766557307/Put/seqid=0 2024-11-16T14:15:59,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742187_1363 (size=12001) 2024-11-16T14:15:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T14:15:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:15:59,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:15:59,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766619503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766619504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:59,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766619505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:59,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766619506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:59,520 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in f43c903d1ca08ce04b9ee6d4b3aa8643 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:15:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:59,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766619519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:59,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766619626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766619626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766619626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:59,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766619635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:59,690 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/5bd852b378aa4fb1b379acb3dab11b2b 2024-11-16T14:15:59,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/ed75ad9ce1ef404c98cabeb617ebf49a is 50, key is test_row_0/C:col10/1731766557307/Put/seqid=0 2024-11-16T14:15:59,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742188_1364 (size=12001) 2024-11-16T14:15:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T14:15:59,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766619836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:59,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766619845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766619845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:15:59,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:15:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766619856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:15:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:15:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,101 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/ed75ad9ce1ef404c98cabeb617ebf49a 2024-11-16T14:16:00,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/bbf22413c56d47ab94ec868ce5633dd8 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bbf22413c56d47ab94ec868ce5633dd8 2024-11-16T14:16:00,107 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bbf22413c56d47ab94ec868ce5633dd8, entries=150, sequenceid=118, filesize=30.2 K 2024-11-16T14:16:00,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/5bd852b378aa4fb1b379acb3dab11b2b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/5bd852b378aa4fb1b379acb3dab11b2b 2024-11-16T14:16:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,110 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/5bd852b378aa4fb1b379acb3dab11b2b, entries=150, sequenceid=118, filesize=11.7 K 2024-11-16T14:16:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/ed75ad9ce1ef404c98cabeb617ebf49a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/ed75ad9ce1ef404c98cabeb617ebf49a 2024-11-16T14:16:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,115 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/ed75ad9ce1ef404c98cabeb617ebf49a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-16T14:16:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,116 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1267ms, sequenceid=118, compaction requested=true 2024-11-16T14:16:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-16T14:16:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-16T14:16:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-16T14:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4190 sec 2024-11-16T14:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4230 sec 2024-11-16T14:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:16:00,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:16:00,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:00,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:16:00,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:16:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fae3f82fe5474f929029cad3dc075520_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766559503/Put/seqid=0 2024-11-16T14:16:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742189_1365 (size=12254) 2024-11-16T14:16:00,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766620232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766620233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766620234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766620240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766620345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766620345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766620356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766620356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766620565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766620565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766620576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766620576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,602 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:00,604 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fae3f82fe5474f929029cad3dc075520_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fae3f82fe5474f929029cad3dc075520_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:00,605 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/d84ec6cdcc314e3f88aa7eb9b504c59c, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:00,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/d84ec6cdcc314e3f88aa7eb9b504c59c is 175, key is test_row_0/A:col10/1731766559503/Put/seqid=0 2024-11-16T14:16:00,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742190_1366 (size=31055) 2024-11-16T14:16:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T14:16:00,855 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:00,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-16T14:16:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:16:00,859 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:00,860 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:00,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:00,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766620885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766620886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766620896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766620896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:16:01,009 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/d84ec6cdcc314e3f88aa7eb9b504c59c 2024-11-16T14:16:01,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:16:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/f10a68c74493495a8d60e8b6dbc9b5c7 is 50, key is test_row_0/B:col10/1731766559503/Put/seqid=0 2024-11-16T14:16:01,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742191_1367 (size=12101) 2024-11-16T14:16:01,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:16:01,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:01,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:16:01,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:16:01,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:01,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766621406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:01,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/f10a68c74493495a8d60e8b6dbc9b5c7 2024-11-16T14:16:01,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766621415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:01,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766621415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:01,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766621415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:01,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/c962a1314d3c43d9b701fb54b7c4ea14 is 50, key is test_row_0/C:col10/1731766559503/Put/seqid=0 2024-11-16T14:16:01,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742192_1368 (size=12101) 2024-11-16T14:16:01,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:16:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:16:01,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:16:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:16:01,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:01,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:01,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/c962a1314d3c43d9b701fb54b7c4ea14 2024-11-16T14:16:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/d84ec6cdcc314e3f88aa7eb9b504c59c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/d84ec6cdcc314e3f88aa7eb9b504c59c 2024-11-16T14:16:01,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/d84ec6cdcc314e3f88aa7eb9b504c59c, entries=150, sequenceid=131, filesize=30.3 K 2024-11-16T14:16:01,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/f10a68c74493495a8d60e8b6dbc9b5c7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/f10a68c74493495a8d60e8b6dbc9b5c7 2024-11-16T14:16:01,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/f10a68c74493495a8d60e8b6dbc9b5c7, entries=150, sequenceid=131, filesize=11.8 K 2024-11-16T14:16:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/c962a1314d3c43d9b701fb54b7c4ea14 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/c962a1314d3c43d9b701fb54b7c4ea14 2024-11-16T14:16:01,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/c962a1314d3c43d9b701fb54b7c4ea14, entries=150, sequenceid=131, filesize=11.8 K 2024-11-16T14:16:01,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1665ms, sequenceid=131, compaction requested=true 2024-11-16T14:16:01,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:01,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:01,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:01,845 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T14:16:01,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:01,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:01,845 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T14:16:01,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:01,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:01,846 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60208 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T14:16:01,846 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 163572 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T14:16:01,846 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/B is initiating minor compaction (all files) 2024-11-16T14:16:01,846 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/A is initiating minor compaction (all files) 2024-11-16T14:16:01,846 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/B in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,846 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/A in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,846 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/0901b2c119c94ee19a80555de1f464e3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/14f3c0cdaf0d4a4185fe8b7caea280d2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ea0d1ef8fabc4fd58a39af376083f434, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/5bd852b378aa4fb1b379acb3dab11b2b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/f10a68c74493495a8d60e8b6dbc9b5c7] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=58.8 K 2024-11-16T14:16:01,846 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b843b895cb9e45b4a9befd3f46fd83fe, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5132bed147c846cd82c32e06ca14a374, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/27e717b3d41a45cb996891eebda1d078, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bbf22413c56d47ab94ec868ce5633dd8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/d84ec6cdcc314e3f88aa7eb9b504c59c] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=159.7 K 2024-11-16T14:16:01,846 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,846 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b843b895cb9e45b4a9befd3f46fd83fe, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5132bed147c846cd82c32e06ca14a374, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/27e717b3d41a45cb996891eebda1d078, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bbf22413c56d47ab94ec868ce5633dd8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/d84ec6cdcc314e3f88aa7eb9b504c59c] 2024-11-16T14:16:01,847 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0901b2c119c94ee19a80555de1f464e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731766554696 2024-11-16T14:16:01,847 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b843b895cb9e45b4a9befd3f46fd83fe, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731766554696 2024-11-16T14:16:01,847 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 14f3c0cdaf0d4a4185fe8b7caea280d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731766555082 2024-11-16T14:16:01,847 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5132bed147c846cd82c32e06ca14a374, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731766555082 2024-11-16T14:16:01,847 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ea0d1ef8fabc4fd58a39af376083f434, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731766555279 2024-11-16T14:16:01,848 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27e717b3d41a45cb996891eebda1d078, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731766555279 2024-11-16T14:16:01,848 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd852b378aa4fb1b379acb3dab11b2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731766557298 2024-11-16T14:16:01,848 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbf22413c56d47ab94ec868ce5633dd8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731766557298 2024-11-16T14:16:01,848 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d84ec6cdcc314e3f88aa7eb9b504c59c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731766559503 2024-11-16T14:16:01,848 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f10a68c74493495a8d60e8b6dbc9b5c7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731766559503 2024-11-16T14:16:01,857 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:01,859 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111649adc6671db147cd8d7de3e9e4d05de3_f43c903d1ca08ce04b9ee6d4b3aa8643 store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:01,860 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#B#compaction#315 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:01,860 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/ea3a8f99b6ec48238a59de7c9b7caa64 is 50, key is test_row_0/B:col10/1731766559503/Put/seqid=0 2024-11-16T14:16:01,862 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111649adc6671db147cd8d7de3e9e4d05de3_f43c903d1ca08ce04b9ee6d4b3aa8643, store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:01,862 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111649adc6671db147cd8d7de3e9e4d05de3_f43c903d1ca08ce04b9ee6d4b3aa8643 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:01,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742193_1369 (size=12375) 2024-11-16T14:16:01,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742194_1370 (size=4469) 2024-11-16T14:16:01,877 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#A#compaction#316 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:01,878 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/6a10cda4c00e488aaaab994f894daeb7 is 175, key is test_row_0/A:col10/1731766559503/Put/seqid=0 2024-11-16T14:16:01,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742195_1371 (size=31329) 2024-11-16T14:16:01,889 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/6a10cda4c00e488aaaab994f894daeb7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6a10cda4c00e488aaaab994f894daeb7 2024-11-16T14:16:01,895 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/A of f43c903d1ca08ce04b9ee6d4b3aa8643 into 6a10cda4c00e488aaaab994f894daeb7(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:01,895 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:01,895 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/A, priority=11, startTime=1731766561845; duration=0sec 2024-11-16T14:16:01,895 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:01,895 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:A 2024-11-16T14:16:01,895 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T14:16:01,897 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60208 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T14:16:01,897 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/C is initiating minor compaction (all files) 2024-11-16T14:16:01,897 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/C in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,897 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f93274cec41c4079b5dc8564a6f0c161, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f7eff9b39a494b5ca6ecdd9eda6f22ba, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/1c714d99a2bd49ac871f3202a1718302, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/ed75ad9ce1ef404c98cabeb617ebf49a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/c962a1314d3c43d9b701fb54b7c4ea14] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=58.8 K 2024-11-16T14:16:01,898 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f93274cec41c4079b5dc8564a6f0c161, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731766554696 2024-11-16T14:16:01,898 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7eff9b39a494b5ca6ecdd9eda6f22ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731766555082 2024-11-16T14:16:01,898 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c714d99a2bd49ac871f3202a1718302, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731766555279 2024-11-16T14:16:01,899 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed75ad9ce1ef404c98cabeb617ebf49a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731766557298 2024-11-16T14:16:01,899 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c962a1314d3c43d9b701fb54b7c4ea14, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731766559503 2024-11-16T14:16:01,911 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#C#compaction#317 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:01,911 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/9738a9d63f094aa3a8265534709dee1c is 50, key is test_row_0/C:col10/1731766559503/Put/seqid=0 2024-11-16T14:16:01,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742196_1372 (size=12375) 2024-11-16T14:16:01,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:16:01,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:01,931 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:16:01,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:16:01,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:01,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:16:01,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:01,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:16:01,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:01,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168b4c432d347f4bbfb943fa8a2d964f7a_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766560239/Put/seqid=0 2024-11-16T14:16:01,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742197_1373 (size=12304) 2024-11-16T14:16:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:16:02,274 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/ea3a8f99b6ec48238a59de7c9b7caa64 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ea3a8f99b6ec48238a59de7c9b7caa64 2024-11-16T14:16:02,278 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/B of f43c903d1ca08ce04b9ee6d4b3aa8643 into ea3a8f99b6ec48238a59de7c9b7caa64(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:02,278 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:02,279 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/B, priority=11, startTime=1731766561845; duration=0sec 2024-11-16T14:16:02,279 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:02,279 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:B 2024-11-16T14:16:02,325 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/9738a9d63f094aa3a8265534709dee1c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/9738a9d63f094aa3a8265534709dee1c 2024-11-16T14:16:02,329 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/C of f43c903d1ca08ce04b9ee6d4b3aa8643 into 9738a9d63f094aa3a8265534709dee1c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:02,329 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:02,329 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/C, priority=11, startTime=1731766561845; duration=0sec 2024-11-16T14:16:02,329 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:02,329 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:C 2024-11-16T14:16:02,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:02,363 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168b4c432d347f4bbfb943fa8a2d964f7a_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168b4c432d347f4bbfb943fa8a2d964f7a_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:02,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/8a34d736c96b4e04bd682e456c14fa6b, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:02,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/8a34d736c96b4e04bd682e456c14fa6b is 175, key is test_row_0/A:col10/1731766560239/Put/seqid=0 2024-11-16T14:16:02,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742198_1374 (size=31105) 2024-11-16T14:16:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:02,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:02,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766622445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:02,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766622448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:02,457 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in f43c903d1ca08ce04b9ee6d4b3aa8643 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:02,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766622449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:02,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:02,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766622457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:02,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:02,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766622565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:02,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:02,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766622565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:02,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:02,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766622576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:02,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:02,768 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/8a34d736c96b4e04bd682e456c14fa6b 2024-11-16T14:16:02,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/8bd1d3de1413482e8e15649bbb72d115 is 50, key is test_row_0/B:col10/1731766560239/Put/seqid=0 2024-11-16T14:16:02,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742199_1375 (size=12151) 2024-11-16T14:16:02,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766622775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:02,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766622776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766622786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:16:03,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766623086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766623086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766623095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,178 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/8bd1d3de1413482e8e15649bbb72d115 2024-11-16T14:16:03,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/edc66f64f16341c8aef6894ece7a1345 is 50, key is test_row_0/C:col10/1731766560239/Put/seqid=0 2024-11-16T14:16:03,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742200_1376 (size=12151) 2024-11-16T14:16:03,193 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/edc66f64f16341c8aef6894ece7a1345 2024-11-16T14:16:03,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/8a34d736c96b4e04bd682e456c14fa6b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/8a34d736c96b4e04bd682e456c14fa6b 2024-11-16T14:16:03,200 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/8a34d736c96b4e04bd682e456c14fa6b, entries=150, sequenceid=151, filesize=30.4 K 2024-11-16T14:16:03,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/8bd1d3de1413482e8e15649bbb72d115 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/8bd1d3de1413482e8e15649bbb72d115 2024-11-16T14:16:03,203 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/8bd1d3de1413482e8e15649bbb72d115, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T14:16:03,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/edc66f64f16341c8aef6894ece7a1345 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/edc66f64f16341c8aef6894ece7a1345 2024-11-16T14:16:03,206 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/edc66f64f16341c8aef6894ece7a1345, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T14:16:03,207 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1276ms, sequenceid=151, compaction requested=false 2024-11-16T14:16:03,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:03,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:03,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-16T14:16:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-16T14:16:03,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-16T14:16:03,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3480 sec 2024-11-16T14:16:03,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3540 sec 2024-11-16T14:16:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:03,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T14:16:03,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:16:03,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:03,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:16:03,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:03,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:16:03,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:03,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116db77945620674220896980ace6da006c_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766562447/Put/seqid=0 2024-11-16T14:16:03,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742201_1377 (size=14794) 2024-11-16T14:16:03,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766623621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766623623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766623625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766623632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766623736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766623746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766623746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766623746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766623956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766623956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766623956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:03,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:03,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766623965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:03,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:03,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:04,016 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:04,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116db77945620674220896980ace6da006c_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116db77945620674220896980ace6da006c_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:04,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/6306a5ce2cf64887815d1b4117de974d, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:04,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/6306a5ce2cf64887815d1b4117de974d is 175, key is test_row_0/A:col10/1731766562447/Put/seqid=0 2024-11-16T14:16:04,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742202_1378 (size=39749) 2024-11-16T14:16:04,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766624265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:04,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766624265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766624266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:04,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766624276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:04,423 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/6306a5ce2cf64887815d1b4117de974d 2024-11-16T14:16:04,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/193ea896dacf4876aad1f13a1a6d4806 is 50, key is test_row_0/B:col10/1731766562447/Put/seqid=0 2024-11-16T14:16:04,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742203_1379 (size=12151) 2024-11-16T14:16:04,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766624475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:04,479 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in f43c903d1ca08ce04b9ee6d4b3aa8643 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:04,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:04,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766624776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:04,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766624786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:04,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766624786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:04,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766624795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:04,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/193ea896dacf4876aad1f13a1a6d4806 2024-11-16T14:16:04,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/9028ba478a754d3fa4dab192edd60a6e is 50, key is test_row_0/C:col10/1731766562447/Put/seqid=0 2024-11-16T14:16:04,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742204_1380 (size=12151) 2024-11-16T14:16:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:16:05,016 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:05,016 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-16T14:16:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:16:05,018 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:05,018 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:05,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:16:05,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:16:05,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:05,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:05,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:05,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:05,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:05,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/9028ba478a754d3fa4dab192edd60a6e 2024-11-16T14:16:05,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/6306a5ce2cf64887815d1b4117de974d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6306a5ce2cf64887815d1b4117de974d 2024-11-16T14:16:05,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6306a5ce2cf64887815d1b4117de974d, entries=200, sequenceid=167, filesize=38.8 K 2024-11-16T14:16:05,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/193ea896dacf4876aad1f13a1a6d4806 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/193ea896dacf4876aad1f13a1a6d4806 2024-11-16T14:16:05,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/193ea896dacf4876aad1f13a1a6d4806, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T14:16:05,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/9028ba478a754d3fa4dab192edd60a6e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/9028ba478a754d3fa4dab192edd60a6e 2024-11-16T14:16:05,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/9028ba478a754d3fa4dab192edd60a6e, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T14:16:05,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1701ms, sequenceid=167, compaction requested=true 2024-11-16T14:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:05,263 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:05,263 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:05,264 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:05,264 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:05,264 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/A is initiating minor compaction (all files) 2024-11-16T14:16:05,264 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/B is initiating minor compaction (all files) 2024-11-16T14:16:05,264 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/A in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:05,264 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/B in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:05,264 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6a10cda4c00e488aaaab994f894daeb7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/8a34d736c96b4e04bd682e456c14fa6b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6306a5ce2cf64887815d1b4117de974d] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=99.8 K 2024-11-16T14:16:05,264 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ea3a8f99b6ec48238a59de7c9b7caa64, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/8bd1d3de1413482e8e15649bbb72d115, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/193ea896dacf4876aad1f13a1a6d4806] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=35.8 K 2024-11-16T14:16:05,264 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:05,264 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6a10cda4c00e488aaaab994f894daeb7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/8a34d736c96b4e04bd682e456c14fa6b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6306a5ce2cf64887815d1b4117de974d] 2024-11-16T14:16:05,264 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ea3a8f99b6ec48238a59de7c9b7caa64, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731766559503 2024-11-16T14:16:05,264 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a10cda4c00e488aaaab994f894daeb7, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731766559503 2024-11-16T14:16:05,264 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bd1d3de1413482e8e15649bbb72d115, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731766560217 2024-11-16T14:16:05,264 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a34d736c96b4e04bd682e456c14fa6b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731766560217 2024-11-16T14:16:05,265 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6306a5ce2cf64887815d1b4117de974d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731766562447 2024-11-16T14:16:05,265 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 193ea896dacf4876aad1f13a1a6d4806, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731766562447 2024-11-16T14:16:05,269 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:05,271 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#B#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:05,272 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/51a9b538192142e38573963e1feb2020 is 50, key is test_row_0/B:col10/1731766562447/Put/seqid=0 2024-11-16T14:16:05,272 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116780f88591af64a979d9f7d085e458b0a_f43c903d1ca08ce04b9ee6d4b3aa8643 store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:05,275 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116780f88591af64a979d9f7d085e458b0a_f43c903d1ca08ce04b9ee6d4b3aa8643, store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:05,275 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116780f88591af64a979d9f7d085e458b0a_f43c903d1ca08ce04b9ee6d4b3aa8643 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:05,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742205_1381 (size=12527) 2024-11-16T14:16:05,283 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/51a9b538192142e38573963e1feb2020 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/51a9b538192142e38573963e1feb2020 2024-11-16T14:16:05,287 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/B of f43c903d1ca08ce04b9ee6d4b3aa8643 into 51a9b538192142e38573963e1feb2020(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:05,287 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:05,287 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/B, priority=13, startTime=1731766565263; duration=0sec 2024-11-16T14:16:05,287 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:05,287 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:B 2024-11-16T14:16:05,287 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:05,288 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:05,288 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/C is initiating minor compaction (all files) 2024-11-16T14:16:05,288 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/C in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:05,288 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/9738a9d63f094aa3a8265534709dee1c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/edc66f64f16341c8aef6894ece7a1345, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/9028ba478a754d3fa4dab192edd60a6e] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=35.8 K 2024-11-16T14:16:05,289 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9738a9d63f094aa3a8265534709dee1c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731766559503 2024-11-16T14:16:05,289 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting edc66f64f16341c8aef6894ece7a1345, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731766560217 2024-11-16T14:16:05,290 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9028ba478a754d3fa4dab192edd60a6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731766562447 2024-11-16T14:16:05,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742206_1382 (size=4469) 2024-11-16T14:16:05,299 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#C#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:05,300 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/f1171d098ab24b2da7b5568f35bd16d2 is 50, key is test_row_0/C:col10/1731766562447/Put/seqid=0 2024-11-16T14:16:05,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742207_1383 (size=12527) 2024-11-16T14:16:05,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:16:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:05,323 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T14:16:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:16:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:16:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:16:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b08684f5bf434be3a181da0be055379d_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766563608/Put/seqid=0 2024-11-16T14:16:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:16:05,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742208_1384 (size=12304) 2024-11-16T14:16:05,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:05,344 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b08684f5bf434be3a181da0be055379d_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b08684f5bf434be3a181da0be055379d_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:05,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/1883cac72c6d44fabeb4e84a1f482cca, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:05,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/1883cac72c6d44fabeb4e84a1f482cca is 175, key is test_row_0/A:col10/1731766563608/Put/seqid=0 2024-11-16T14:16:05,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742209_1385 (size=31105) 2024-11-16T14:16:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:16:05,692 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#A#compaction#324 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:05,692 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/0d3463f8b6d24556ab39891c6d623dbc is 175, key is test_row_0/A:col10/1731766562447/Put/seqid=0 2024-11-16T14:16:05,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742210_1386 (size=31481) 2024-11-16T14:16:05,708 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/f1171d098ab24b2da7b5568f35bd16d2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f1171d098ab24b2da7b5568f35bd16d2 2024-11-16T14:16:05,712 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/C of f43c903d1ca08ce04b9ee6d4b3aa8643 into f1171d098ab24b2da7b5568f35bd16d2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:05,712 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:05,712 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/C, priority=13, startTime=1731766565263; duration=0sec 2024-11-16T14:16:05,712 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:05,712 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:C 2024-11-16T14:16:05,753 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/1883cac72c6d44fabeb4e84a1f482cca 2024-11-16T14:16:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/5a9e609305b74ce888d13710214d71a1 is 50, key is test_row_0/B:col10/1731766563608/Put/seqid=0 2024-11-16T14:16:05,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742211_1387 (size=12151) 2024-11-16T14:16:05,771 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/5a9e609305b74ce888d13710214d71a1 2024-11-16T14:16:05,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/cba9402b7d3242ae8ac02ed5e54373ba is 50, key is test_row_0/C:col10/1731766563608/Put/seqid=0 2024-11-16T14:16:05,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742212_1388 (size=12151) 2024-11-16T14:16:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:05,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:05,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766625827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:05,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766625828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:05,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766625829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:05,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766625831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:05,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766625936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:05,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766625945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766625945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:05,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766625945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,102 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/0d3463f8b6d24556ab39891c6d623dbc as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/0d3463f8b6d24556ab39891c6d623dbc 2024-11-16T14:16:06,106 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/A of f43c903d1ca08ce04b9ee6d4b3aa8643 into 0d3463f8b6d24556ab39891c6d623dbc(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:06,106 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:06,106 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/A, priority=13, startTime=1731766565263; duration=0sec 2024-11-16T14:16:06,106 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:06,106 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:A 2024-11-16T14:16:06,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766626146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:16:06,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766626165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766626165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766626165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,183 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/cba9402b7d3242ae8ac02ed5e54373ba 2024-11-16T14:16:06,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/1883cac72c6d44fabeb4e84a1f482cca as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1883cac72c6d44fabeb4e84a1f482cca 2024-11-16T14:16:06,192 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1883cac72c6d44fabeb4e84a1f482cca, entries=150, sequenceid=186, filesize=30.4 K 2024-11-16T14:16:06,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/5a9e609305b74ce888d13710214d71a1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/5a9e609305b74ce888d13710214d71a1 2024-11-16T14:16:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,195 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/5a9e609305b74ce888d13710214d71a1, entries=150, sequenceid=186, filesize=11.9 K 2024-11-16T14:16:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/cba9402b7d3242ae8ac02ed5e54373ba as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/cba9402b7d3242ae8ac02ed5e54373ba 2024-11-16T14:16:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,200 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/cba9402b7d3242ae8ac02ed5e54373ba, entries=150, sequenceid=186, filesize=11.9 K 2024-11-16T14:16:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,201 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 878ms, sequenceid=186, compaction requested=false 2024-11-16T14:16:06,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:06,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-16T14:16:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-16T14:16:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-16T14:16:06,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1840 sec 2024-11-16T14:16:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.1880 sec 2024-11-16T14:16:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:06,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T14:16:06,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:16:06,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:06,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:16:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:16:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165c1ccef2e1d04167acbf709a5d492dbd_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_1/A:col10/1731766565830/Put/seqid=0 2024-11-16T14:16:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742213_1389 (size=12304) 2024-11-16T14:16:06,542 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,550 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165c1ccef2e1d04167acbf709a5d492dbd_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165c1ccef2e1d04167acbf709a5d492dbd_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:06,551 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/e5cc99e3493847adbb262959a63fa01d, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:06,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/e5cc99e3493847adbb262959a63fa01d is 175, key is test_row_1/A:col10/1731766565830/Put/seqid=0 2024-11-16T14:16:06,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742214_1390 (size=31101) 2024-11-16T14:16:06,562 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/e5cc99e3493847adbb262959a63fa01d 2024-11-16T14:16:06,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766626550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766626554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/cea403ce9fe54e08a50366dfa49ed00c is 50, key is test_row_1/B:col10/1731766565830/Put/seqid=0 2024-11-16T14:16:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766626572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766626572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742215_1391 (size=9757) 2024-11-16T14:16:06,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/cea403ce9fe54e08a50366dfa49ed00c 2024-11-16T14:16:06,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/02e2f6385b8b40f085485649df8d22b1 is 50, key is test_row_1/C:col10/1731766565830/Put/seqid=0 2024-11-16T14:16:06,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742216_1392 (size=9757) 2024-11-16T14:16:06,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/02e2f6385b8b40f085485649df8d22b1 2024-11-16T14:16:06,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/e5cc99e3493847adbb262959a63fa01d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/e5cc99e3493847adbb262959a63fa01d 2024-11-16T14:16:06,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/e5cc99e3493847adbb262959a63fa01d, entries=150, sequenceid=203, filesize=30.4 K 2024-11-16T14:16:06,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/cea403ce9fe54e08a50366dfa49ed00c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/cea403ce9fe54e08a50366dfa49ed00c 2024-11-16T14:16:06,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/cea403ce9fe54e08a50366dfa49ed00c, entries=100, sequenceid=203, filesize=9.5 K 2024-11-16T14:16:06,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/02e2f6385b8b40f085485649df8d22b1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/02e2f6385b8b40f085485649df8d22b1 2024-11-16T14:16:06,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/02e2f6385b8b40f085485649df8d22b1, entries=100, sequenceid=203, filesize=9.5 K 2024-11-16T14:16:06,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 149ms, sequenceid=203, compaction requested=true 2024-11-16T14:16:06,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:06,635 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:06,636 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:06,636 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/A is initiating minor compaction (all files) 2024-11-16T14:16:06,636 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/A in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:06,636 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/0d3463f8b6d24556ab39891c6d623dbc, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1883cac72c6d44fabeb4e84a1f482cca, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/e5cc99e3493847adbb262959a63fa01d] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=91.5 K 2024-11-16T14:16:06,636 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:06,636 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/0d3463f8b6d24556ab39891c6d623dbc, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1883cac72c6d44fabeb4e84a1f482cca, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/e5cc99e3493847adbb262959a63fa01d] 2024-11-16T14:16:06,637 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d3463f8b6d24556ab39891c6d623dbc, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731766562447 2024-11-16T14:16:06,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:06,637 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1883cac72c6d44fabeb4e84a1f482cca, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731766563608 2024-11-16T14:16:06,637 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5cc99e3493847adbb262959a63fa01d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766565828 2024-11-16T14:16:06,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:06,638 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:06,639 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:06,639 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/B is initiating minor compaction (all files) 2024-11-16T14:16:06,639 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/B in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:06,639 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/51a9b538192142e38573963e1feb2020, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/5a9e609305b74ce888d13710214d71a1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/cea403ce9fe54e08a50366dfa49ed00c] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=33.6 K 2024-11-16T14:16:06,640 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 51a9b538192142e38573963e1feb2020, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731766562447 2024-11-16T14:16:06,640 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a9e609305b74ce888d13710214d71a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731766563608 2024-11-16T14:16:06,641 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting cea403ce9fe54e08a50366dfa49ed00c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766565830 2024-11-16T14:16:06,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:06,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:06,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:06,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:06,648 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:06,654 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#B#compaction#334 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:06,654 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/3de44ae934dc437b89345bf13acd0a71 is 50, key is test_row_0/B:col10/1731766563608/Put/seqid=0 2024-11-16T14:16:06,657 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116b35c88eb7b89496998395c5ff40f9560_f43c903d1ca08ce04b9ee6d4b3aa8643 store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:06,658 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116b35c88eb7b89496998395c5ff40f9560_f43c903d1ca08ce04b9ee6d4b3aa8643, store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:06,659 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b35c88eb7b89496998395c5ff40f9560_f43c903d1ca08ce04b9ee6d4b3aa8643 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:06,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742217_1393 (size=12629) 2024-11-16T14:16:06,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742218_1394 (size=4469) 2024-11-16T14:16:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:06,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:16:06,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:16:06,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:06,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:16:06,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:06,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:16:06,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:06,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f706cef0846c4a90a6572b140d91a503_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766566568/Put/seqid=0 2024-11-16T14:16:06,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742219_1395 (size=14794) 2024-11-16T14:16:06,709 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:06,714 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f706cef0846c4a90a6572b140d91a503_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f706cef0846c4a90a6572b140d91a503_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:06,715 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/bb86e06bb41d412293713115a21d47aa, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:06,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/bb86e06bb41d412293713115a21d47aa is 175, key is test_row_0/A:col10/1731766566568/Put/seqid=0 2024-11-16T14:16:06,720 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:16:06,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742220_1396 (size=39749) 2024-11-16T14:16:06,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766626719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766626720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766626722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766626735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766626846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766626846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766626846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766626855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766627056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:07,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,066 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#A#compaction#333 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:07,067 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/7ba1032528174d5fa9c90a954f9aecce is 175, key is test_row_0/A:col10/1731766563608/Put/seqid=0 2024-11-16T14:16:07,071 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/3de44ae934dc437b89345bf13acd0a71 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/3de44ae934dc437b89345bf13acd0a71 2024-11-16T14:16:07,074 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/B of f43c903d1ca08ce04b9ee6d4b3aa8643 into 3de44ae934dc437b89345bf13acd0a71(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:07,074 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:07,075 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/B, priority=13, startTime=1731766566638; duration=0sec 2024-11-16T14:16:07,075 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:07,075 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:B 2024-11-16T14:16:07,075 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:07,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766627065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766627065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,076 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:07,076 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/C is initiating minor compaction (all files) 2024-11-16T14:16:07,076 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/C in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:07,076 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f1171d098ab24b2da7b5568f35bd16d2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/cba9402b7d3242ae8ac02ed5e54373ba, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/02e2f6385b8b40f085485649df8d22b1] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=33.6 K 2024-11-16T14:16:07,077 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f1171d098ab24b2da7b5568f35bd16d2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731766562447 2024-11-16T14:16:07,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742221_1397 (size=31690) 2024-11-16T14:16:07,078 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting cba9402b7d3242ae8ac02ed5e54373ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731766563608 2024-11-16T14:16:07,078 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 02e2f6385b8b40f085485649df8d22b1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766565830 2024-11-16T14:16:07,081 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/7ba1032528174d5fa9c90a954f9aecce as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/7ba1032528174d5fa9c90a954f9aecce 2024-11-16T14:16:07,084 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/A of f43c903d1ca08ce04b9ee6d4b3aa8643 into 7ba1032528174d5fa9c90a954f9aecce(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:07,084 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:07,084 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/A, priority=13, startTime=1731766566635; duration=0sec 2024-11-16T14:16:07,084 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:07,084 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:A 2024-11-16T14:16:07,085 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:07,085 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/f76b82a141c2440f9c65c0de93408835 is 50, key is test_row_0/C:col10/1731766563608/Put/seqid=0 2024-11-16T14:16:07,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766627076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:07,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742222_1398 (size=12629) 2024-11-16T14:16:07,094 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/f76b82a141c2440f9c65c0de93408835 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f76b82a141c2440f9c65c0de93408835 2024-11-16T14:16:07,097 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/C of f43c903d1ca08ce04b9ee6d4b3aa8643 into f76b82a141c2440f9c65c0de93408835(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:07,097 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:07,097 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/C, priority=13, startTime=1731766566641; duration=0sec 2024-11-16T14:16:07,097 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:07,098 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:C 2024-11-16T14:16:07,132 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/bb86e06bb41d412293713115a21d47aa 2024-11-16T14:16:07,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/b11b805ce71c44f384a4abdcf45899ae is 50, key is test_row_0/B:col10/1731766566568/Put/seqid=0 2024-11-16T14:16:07,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742223_1399 (size=12151) 2024-11-16T14:16:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:16:07,165 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:07,166 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-16T14:16:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:16:07,167 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:07,168 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:07,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:16:07,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T14:16:07,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:07,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:07,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:07,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766627376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766627385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766627385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766627395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T14:16:07,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:07,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:07,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:07,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:16:07,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/b11b805ce71c44f384a4abdcf45899ae 2024-11-16T14:16:07,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/96d4d059b0184685858e5ff581542752 is 50, key is test_row_0/C:col10/1731766566568/Put/seqid=0 2024-11-16T14:16:07,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742224_1400 (size=12151) 2024-11-16T14:16:07,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T14:16:07,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:07,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:07,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:07,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T14:16:07,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:07,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:07,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:07,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:16:07,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766627885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766627906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:07,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766627906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766627916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:07,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:07,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:07,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T14:16:07,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:07,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:07,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:07,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:07,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/96d4d059b0184685858e5ff581542752 2024-11-16T14:16:07,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/bb86e06bb41d412293713115a21d47aa as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bb86e06bb41d412293713115a21d47aa 2024-11-16T14:16:07,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bb86e06bb41d412293713115a21d47aa, entries=200, sequenceid=222, filesize=38.8 K 2024-11-16T14:16:07,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/b11b805ce71c44f384a4abdcf45899ae as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/b11b805ce71c44f384a4abdcf45899ae 2024-11-16T14:16:07,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/b11b805ce71c44f384a4abdcf45899ae, entries=150, sequenceid=222, filesize=11.9 K 2024-11-16T14:16:07,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/96d4d059b0184685858e5ff581542752 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/96d4d059b0184685858e5ff581542752 2024-11-16T14:16:07,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/96d4d059b0184685858e5ff581542752, entries=150, sequenceid=222, filesize=11.9 K 2024-11-16T14:16:07,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1284ms, sequenceid=222, compaction requested=false 2024-11-16T14:16:07,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:08,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T14:16:08,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:08,086 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:16:08,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:16:08,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:08,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:16:08,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:08,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:16:08,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:08,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161e89c3d20a68496ba8721c354eec51a7_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766566722/Put/seqid=0 2024-11-16T14:16:08,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742225_1401 (size=12304) 2024-11-16T14:16:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:16:08,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:08,501 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161e89c3d20a68496ba8721c354eec51a7_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161e89c3d20a68496ba8721c354eec51a7_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:08,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/5a37da73e7a64a2187490995cbbc6314, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:08,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/5a37da73e7a64a2187490995cbbc6314 is 175, key is test_row_0/A:col10/1731766566722/Put/seqid=0 2024-11-16T14:16:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:08,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:08,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742226_1402 (size=31105) 2024-11-16T14:16:08,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766628600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:08,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766628715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:08,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766628895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:08,905 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in f43c903d1ca08ce04b9ee6d4b3aa8643 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:08,905 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/5a37da73e7a64a2187490995cbbc6314 2024-11-16T14:16:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:08,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/e468738add594e7683f77f2e7da4caa5 is 50, key is test_row_0/B:col10/1731766566722/Put/seqid=0 2024-11-16T14:16:08,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742227_1403 (size=12151) 2024-11-16T14:16:08,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766628925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:08,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766628926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:08,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766628926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:08,932 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in f43c903d1ca08ce04b9ee6d4b3aa8643 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,932 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in f43c903d1ca08ce04b9ee6d4b3aa8643 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:08,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:08,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:08,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766628936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:08,948 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in f43c903d1ca08ce04b9ee6d4b3aa8643 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:09,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766629245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:09,315 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/e468738add594e7683f77f2e7da4caa5 2024-11-16T14:16:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:16:09,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/a0ee6ee7aec946a588a39bcceaf9c085 is 50, key is test_row_0/C:col10/1731766566722/Put/seqid=0 2024-11-16T14:16:09,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742228_1404 (size=12151) 2024-11-16T14:16:09,731 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/a0ee6ee7aec946a588a39bcceaf9c085 2024-11-16T14:16:09,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/5a37da73e7a64a2187490995cbbc6314 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5a37da73e7a64a2187490995cbbc6314 2024-11-16T14:16:09,738 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5a37da73e7a64a2187490995cbbc6314, entries=150, sequenceid=238, filesize=30.4 K 2024-11-16T14:16:09,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/e468738add594e7683f77f2e7da4caa5 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/e468738add594e7683f77f2e7da4caa5 2024-11-16T14:16:09,741 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/e468738add594e7683f77f2e7da4caa5, entries=150, sequenceid=238, filesize=11.9 K 2024-11-16T14:16:09,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/a0ee6ee7aec946a588a39bcceaf9c085 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/a0ee6ee7aec946a588a39bcceaf9c085 2024-11-16T14:16:09,747 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/a0ee6ee7aec946a588a39bcceaf9c085, entries=150, sequenceid=238, filesize=11.9 K 2024-11-16T14:16:09,748 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1661ms, sequenceid=238, compaction requested=true 2024-11-16T14:16:09,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:09,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:09,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-16T14:16:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-16T14:16:09,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-16T14:16:09,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5810 sec 2024-11-16T14:16:09,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.5850 sec 2024-11-16T14:16:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:09,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:16:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:16:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:16:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:16:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:09,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111665e2ba33c7c3418ba83dec6d012860c8_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766568564/Put/seqid=0 2024-11-16T14:16:09,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742229_1405 (size=14844) 2024-11-16T14:16:09,776 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:09,778 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111665e2ba33c7c3418ba83dec6d012860c8_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111665e2ba33c7c3418ba83dec6d012860c8_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:09,781 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/c55fd98154504a0bb76283dfcb16d776, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:09,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/c55fd98154504a0bb76283dfcb16d776 is 175, key is test_row_0/A:col10/1731766568564/Put/seqid=0 2024-11-16T14:16:09,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742230_1406 (size=39799) 2024-11-16T14:16:09,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766629835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:09,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766629955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:10,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766630175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:10,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:10,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:10,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:10,185 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/c55fd98154504a0bb76283dfcb16d776 2024-11-16T14:16:10,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/ff4ca54eed244cf5b6fa9f720660a5e3 is 50, key is test_row_0/B:col10/1731766568564/Put/seqid=0 2024-11-16T14:16:10,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742231_1407 (size=12201) 2024-11-16T14:16:10,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766630485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:10,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/ff4ca54eed244cf5b6fa9f720660a5e3 2024-11-16T14:16:10,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/15b683fe38af4ccd9aebc4b23c69617f is 50, key is test_row_0/C:col10/1731766568564/Put/seqid=0 2024-11-16T14:16:10,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742232_1408 (size=12201) 2024-11-16T14:16:10,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39876 deadline: 1731766630925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:10,932 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in f43c903d1ca08ce04b9ee6d4b3aa8643 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:10,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39844 deadline: 1731766630946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:10,950 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in f43c903d1ca08ce04b9ee6d4b3aa8643 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:10,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:10,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:10,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:10,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39880 deadline: 1731766630955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:10,962 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in f43c903d1ca08ce04b9ee6d4b3aa8643 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:10,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1731766630965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:10,975 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in f43c903d1ca08ce04b9ee6d4b3aa8643 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:10,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:10,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:10,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:11,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766631006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:11,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/15b683fe38af4ccd9aebc4b23c69617f 2024-11-16T14:16:11,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/c55fd98154504a0bb76283dfcb16d776 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/c55fd98154504a0bb76283dfcb16d776 2024-11-16T14:16:11,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/c55fd98154504a0bb76283dfcb16d776, entries=200, sequenceid=258, filesize=38.9 K 2024-11-16T14:16:11,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/ff4ca54eed244cf5b6fa9f720660a5e3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ff4ca54eed244cf5b6fa9f720660a5e3 2024-11-16T14:16:11,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ff4ca54eed244cf5b6fa9f720660a5e3, entries=150, sequenceid=258, filesize=11.9 K 2024-11-16T14:16:11,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/15b683fe38af4ccd9aebc4b23c69617f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/15b683fe38af4ccd9aebc4b23c69617f 2024-11-16T14:16:11,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/15b683fe38af4ccd9aebc4b23c69617f, entries=150, sequenceid=258, filesize=11.9 K 2024-11-16T14:16:11,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1269ms, sequenceid=258, compaction requested=true 2024-11-16T14:16:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:11,032 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:16:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:11,032 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:16:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:11,034 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142343 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:16:11,034 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:16:11,034 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/A is initiating minor compaction (all files) 2024-11-16T14:16:11,034 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/B is initiating minor compaction (all files) 2024-11-16T14:16:11,034 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/A in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:11,034 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/B in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:11,034 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/7ba1032528174d5fa9c90a954f9aecce, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bb86e06bb41d412293713115a21d47aa, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5a37da73e7a64a2187490995cbbc6314, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/c55fd98154504a0bb76283dfcb16d776] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=139.0 K 2024-11-16T14:16:11,034 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/3de44ae934dc437b89345bf13acd0a71, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/b11b805ce71c44f384a4abdcf45899ae, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/e468738add594e7683f77f2e7da4caa5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ff4ca54eed244cf5b6fa9f720660a5e3] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=48.0 K 2024-11-16T14:16:11,034 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:11,034 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/7ba1032528174d5fa9c90a954f9aecce, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bb86e06bb41d412293713115a21d47aa, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5a37da73e7a64a2187490995cbbc6314, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/c55fd98154504a0bb76283dfcb16d776] 2024-11-16T14:16:11,034 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3de44ae934dc437b89345bf13acd0a71, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766563608 2024-11-16T14:16:11,034 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ba1032528174d5fa9c90a954f9aecce, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766563608 2024-11-16T14:16:11,035 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting b11b805ce71c44f384a4abdcf45899ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731766566544 2024-11-16T14:16:11,035 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb86e06bb41d412293713115a21d47aa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731766566544 2024-11-16T14:16:11,035 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a37da73e7a64a2187490995cbbc6314, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731766566716 2024-11-16T14:16:11,035 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting e468738add594e7683f77f2e7da4caa5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731766566716 2024-11-16T14:16:11,035 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c55fd98154504a0bb76283dfcb16d776, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731766568564 2024-11-16T14:16:11,035 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ff4ca54eed244cf5b6fa9f720660a5e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731766568564 2024-11-16T14:16:11,047 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:11,049 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:11,050 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/1313e91266fc4d19ae68fb7ea6556a58 is 50, key is test_row_0/B:col10/1731766568564/Put/seqid=0 2024-11-16T14:16:11,052 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111677c5341299ec48a0ad16d535ffbb46a3_f43c903d1ca08ce04b9ee6d4b3aa8643 store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:11,054 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111677c5341299ec48a0ad16d535ffbb46a3_f43c903d1ca08ce04b9ee6d4b3aa8643, store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:11,054 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111677c5341299ec48a0ad16d535ffbb46a3_f43c903d1ca08ce04b9ee6d4b3aa8643 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:11,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742233_1409 (size=12815) 2024-11-16T14:16:11,066 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/1313e91266fc4d19ae68fb7ea6556a58 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/1313e91266fc4d19ae68fb7ea6556a58 2024-11-16T14:16:11,075 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/B of f43c903d1ca08ce04b9ee6d4b3aa8643 into 1313e91266fc4d19ae68fb7ea6556a58(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:11,075 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:11,075 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/B, priority=12, startTime=1731766571032; duration=0sec 2024-11-16T14:16:11,075 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:11,075 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:B 2024-11-16T14:16:11,075 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:16:11,076 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:16:11,076 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/C is initiating minor compaction (all files) 2024-11-16T14:16:11,077 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/C in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:11,077 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f76b82a141c2440f9c65c0de93408835, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/96d4d059b0184685858e5ff581542752, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/a0ee6ee7aec946a588a39bcceaf9c085, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/15b683fe38af4ccd9aebc4b23c69617f] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=48.0 K 2024-11-16T14:16:11,077 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f76b82a141c2440f9c65c0de93408835, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766563608 2024-11-16T14:16:11,077 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 96d4d059b0184685858e5ff581542752, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731766566544 2024-11-16T14:16:11,077 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ee6ee7aec946a588a39bcceaf9c085, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731766566716 2024-11-16T14:16:11,078 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 15b683fe38af4ccd9aebc4b23c69617f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731766568564 2024-11-16T14:16:11,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742234_1410 (size=4469) 2024-11-16T14:16:11,113 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#C#compaction#347 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:11,113 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/ade6a2890e4c4f9db9eaeef97e068a6c is 50, key is test_row_0/C:col10/1731766568564/Put/seqid=0 2024-11-16T14:16:11,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742235_1411 (size=12815) 2024-11-16T14:16:11,128 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/ade6a2890e4c4f9db9eaeef97e068a6c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/ade6a2890e4c4f9db9eaeef97e068a6c 2024-11-16T14:16:11,136 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/C of f43c903d1ca08ce04b9ee6d4b3aa8643 into ade6a2890e4c4f9db9eaeef97e068a6c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:11,136 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:11,136 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/C, priority=12, startTime=1731766571032; duration=0sec 2024-11-16T14:16:11,136 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:11,136 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:C 2024-11-16T14:16:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:16:11,336 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:11,336 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-16T14:16:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:16:11,338 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:11,338 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:11,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:16:11,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T14:16:11,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:11,490 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:16:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:16:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:16:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:16:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:11,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166328f7f9ece44ebca784f3e499c58d2d_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766569805/Put/seqid=0 2024-11-16T14:16:11,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742236_1412 (size=12454) 2024-11-16T14:16:11,506 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#A#compaction#346 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:11,506 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/f4f982275fd244e6b864ac2fe75dddd2 is 175, key is test_row_0/A:col10/1731766568564/Put/seqid=0 2024-11-16T14:16:11,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742237_1413 (size=31769) 2024-11-16T14:16:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:16:11,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:11,911 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166328f7f9ece44ebca784f3e499c58d2d_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166328f7f9ece44ebca784f3e499c58d2d_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:11,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/14d9c17406fc460b81a76fc767eed4a7, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:11,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/14d9c17406fc460b81a76fc767eed4a7 is 175, key is test_row_0/A:col10/1731766569805/Put/seqid=0 2024-11-16T14:16:11,915 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/f4f982275fd244e6b864ac2fe75dddd2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/f4f982275fd244e6b864ac2fe75dddd2 2024-11-16T14:16:11,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742238_1414 (size=31255) 2024-11-16T14:16:11,916 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/14d9c17406fc460b81a76fc767eed4a7 2024-11-16T14:16:11,919 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/A of f43c903d1ca08ce04b9ee6d4b3aa8643 into f4f982275fd244e6b864ac2fe75dddd2(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:11,920 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:11,920 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/A, priority=12, startTime=1731766571032; duration=0sec 2024-11-16T14:16:11,920 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:11,920 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:A 2024-11-16T14:16:11,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/bed3318d6b5c4599a957c285e7bf3314 is 50, key is test_row_0/B:col10/1731766569805/Put/seqid=0 2024-11-16T14:16:11,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742239_1415 (size=12301) 2024-11-16T14:16:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:16:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:12,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. as already flushing 2024-11-16T14:16:12,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766632117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:12,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:12,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766632235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:12,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:12,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:12,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:12,326 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/bed3318d6b5c4599a957c285e7bf3314 2024-11-16T14:16:12,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/e9c96d2aaa77483cb908b2a02b1d3f4b is 50, key is test_row_0/C:col10/1731766569805/Put/seqid=0 2024-11-16T14:16:12,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742240_1416 (size=12301) 2024-11-16T14:16:12,381 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-11-16T14:16:12,381 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:12,381 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:12,381 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:12,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:12,382 INFO [Thread-1552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1552. 2024-11-16T14:16:12,382 DEBUG [Thread-1552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:12,382 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:12,382 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:12,382 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:12,386 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-11-16T14:16:12,386 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:12,386 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:12,386 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:12,386 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:12,391 INFO [Thread-1550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1550. 2024-11-16T14:16:12,391 DEBUG [Thread-1550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:12,391 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:12,391 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:12,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:12,392 INFO [Thread-1548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1548. 2024-11-16T14:16:12,392 DEBUG [Thread-1548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:12,392 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:12,392 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:12,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:12,405 WARN [regionserver/04e93f2b5fad:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-16T14:16:12,405 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/795aacd380b341e1b43fe6d0894befcb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b46193a2e4514fa28e6397f7a1344ed9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1cd320732ff446b698e34fc499914896, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b843b895cb9e45b4a9befd3f46fd83fe, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5132bed147c846cd82c32e06ca14a374, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/27e717b3d41a45cb996891eebda1d078, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bbf22413c56d47ab94ec868ce5633dd8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6a10cda4c00e488aaaab994f894daeb7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/d84ec6cdcc314e3f88aa7eb9b504c59c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/8a34d736c96b4e04bd682e456c14fa6b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6306a5ce2cf64887815d1b4117de974d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/0d3463f8b6d24556ab39891c6d623dbc, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1883cac72c6d44fabeb4e84a1f482cca, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/7ba1032528174d5fa9c90a954f9aecce, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/e5cc99e3493847adbb262959a63fa01d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bb86e06bb41d412293713115a21d47aa, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5a37da73e7a64a2187490995cbbc6314, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/c55fd98154504a0bb76283dfcb16d776] to archive 2024-11-16T14:16:12,407 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:16:12,408 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/795aacd380b341e1b43fe6d0894befcb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/795aacd380b341e1b43fe6d0894befcb 2024-11-16T14:16:12,410 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b46193a2e4514fa28e6397f7a1344ed9 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b46193a2e4514fa28e6397f7a1344ed9 2024-11-16T14:16:12,411 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1cd320732ff446b698e34fc499914896 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1cd320732ff446b698e34fc499914896 2024-11-16T14:16:12,412 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b843b895cb9e45b4a9befd3f46fd83fe to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/b843b895cb9e45b4a9befd3f46fd83fe 2024-11-16T14:16:12,414 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5132bed147c846cd82c32e06ca14a374 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5132bed147c846cd82c32e06ca14a374 2024-11-16T14:16:12,415 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/27e717b3d41a45cb996891eebda1d078 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/27e717b3d41a45cb996891eebda1d078 2024-11-16T14:16:12,416 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bbf22413c56d47ab94ec868ce5633dd8 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bbf22413c56d47ab94ec868ce5633dd8 2024-11-16T14:16:12,417 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6a10cda4c00e488aaaab994f894daeb7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6a10cda4c00e488aaaab994f894daeb7 2024-11-16T14:16:12,418 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/d84ec6cdcc314e3f88aa7eb9b504c59c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/d84ec6cdcc314e3f88aa7eb9b504c59c 2024-11-16T14:16:12,419 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/8a34d736c96b4e04bd682e456c14fa6b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/8a34d736c96b4e04bd682e456c14fa6b 2024-11-16T14:16:12,419 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6306a5ce2cf64887815d1b4117de974d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/6306a5ce2cf64887815d1b4117de974d 2024-11-16T14:16:12,420 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/0d3463f8b6d24556ab39891c6d623dbc to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/0d3463f8b6d24556ab39891c6d623dbc 2024-11-16T14:16:12,421 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1883cac72c6d44fabeb4e84a1f482cca to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1883cac72c6d44fabeb4e84a1f482cca 2024-11-16T14:16:12,422 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/7ba1032528174d5fa9c90a954f9aecce to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/7ba1032528174d5fa9c90a954f9aecce 2024-11-16T14:16:12,423 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/e5cc99e3493847adbb262959a63fa01d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/e5cc99e3493847adbb262959a63fa01d 2024-11-16T14:16:12,424 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bb86e06bb41d412293713115a21d47aa to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/bb86e06bb41d412293713115a21d47aa 2024-11-16T14:16:12,425 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5a37da73e7a64a2187490995cbbc6314 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/5a37da73e7a64a2187490995cbbc6314 2024-11-16T14:16:12,426 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/c55fd98154504a0bb76283dfcb16d776 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/c55fd98154504a0bb76283dfcb16d776 2024-11-16T14:16:12,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/8ab0ce8dfacd48ebb0857bf17671db69, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/0251b3e1b26d4cb7887b3795859c3e31, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/0901b2c119c94ee19a80555de1f464e3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/893fdbddecda40b6a2965c0fbb852a57, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/14f3c0cdaf0d4a4185fe8b7caea280d2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ea0d1ef8fabc4fd58a39af376083f434, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/5bd852b378aa4fb1b379acb3dab11b2b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ea3a8f99b6ec48238a59de7c9b7caa64, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/f10a68c74493495a8d60e8b6dbc9b5c7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/8bd1d3de1413482e8e15649bbb72d115, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/51a9b538192142e38573963e1feb2020, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/193ea896dacf4876aad1f13a1a6d4806, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/5a9e609305b74ce888d13710214d71a1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/3de44ae934dc437b89345bf13acd0a71, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/cea403ce9fe54e08a50366dfa49ed00c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/b11b805ce71c44f384a4abdcf45899ae, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/e468738add594e7683f77f2e7da4caa5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ff4ca54eed244cf5b6fa9f720660a5e3] to archive 2024-11-16T14:16:12,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:16:12,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/8ab0ce8dfacd48ebb0857bf17671db69 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/8ab0ce8dfacd48ebb0857bf17671db69 2024-11-16T14:16:12,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/0251b3e1b26d4cb7887b3795859c3e31 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/0251b3e1b26d4cb7887b3795859c3e31 2024-11-16T14:16:12,433 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/0901b2c119c94ee19a80555de1f464e3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/0901b2c119c94ee19a80555de1f464e3 2024-11-16T14:16:12,434 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/893fdbddecda40b6a2965c0fbb852a57 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/893fdbddecda40b6a2965c0fbb852a57 2024-11-16T14:16:12,435 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/14f3c0cdaf0d4a4185fe8b7caea280d2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/14f3c0cdaf0d4a4185fe8b7caea280d2 2024-11-16T14:16:12,436 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ea0d1ef8fabc4fd58a39af376083f434 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ea0d1ef8fabc4fd58a39af376083f434 2024-11-16T14:16:12,437 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/5bd852b378aa4fb1b379acb3dab11b2b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/5bd852b378aa4fb1b379acb3dab11b2b 2024-11-16T14:16:12,438 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ea3a8f99b6ec48238a59de7c9b7caa64 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ea3a8f99b6ec48238a59de7c9b7caa64 2024-11-16T14:16:12,439 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/f10a68c74493495a8d60e8b6dbc9b5c7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/f10a68c74493495a8d60e8b6dbc9b5c7 2024-11-16T14:16:12,441 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/8bd1d3de1413482e8e15649bbb72d115 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/8bd1d3de1413482e8e15649bbb72d115 2024-11-16T14:16:12,442 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/51a9b538192142e38573963e1feb2020 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/51a9b538192142e38573963e1feb2020 2024-11-16T14:16:12,443 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/193ea896dacf4876aad1f13a1a6d4806 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/193ea896dacf4876aad1f13a1a6d4806 2024-11-16T14:16:12,444 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/5a9e609305b74ce888d13710214d71a1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/5a9e609305b74ce888d13710214d71a1 2024-11-16T14:16:12,445 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/3de44ae934dc437b89345bf13acd0a71 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/3de44ae934dc437b89345bf13acd0a71 2024-11-16T14:16:12,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:12,446 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/cea403ce9fe54e08a50366dfa49ed00c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/cea403ce9fe54e08a50366dfa49ed00c 2024-11-16T14:16:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1731766632446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:12,448 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/b11b805ce71c44f384a4abdcf45899ae to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/b11b805ce71c44f384a4abdcf45899ae 2024-11-16T14:16:12,449 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/e468738add594e7683f77f2e7da4caa5 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/e468738add594e7683f77f2e7da4caa5 2024-11-16T14:16:12,450 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ff4ca54eed244cf5b6fa9f720660a5e3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/ff4ca54eed244cf5b6fa9f720660a5e3 2024-11-16T14:16:12,454 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/6a046fe006654047b8be25661a395073, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/02d3c7f7d0bc4df59cfcd1fec042cd8c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f93274cec41c4079b5dc8564a6f0c161, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/8ef0fb9d4fdc4d638c537d003c845bc4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f7eff9b39a494b5ca6ecdd9eda6f22ba, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/1c714d99a2bd49ac871f3202a1718302, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/ed75ad9ce1ef404c98cabeb617ebf49a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/9738a9d63f094aa3a8265534709dee1c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/c962a1314d3c43d9b701fb54b7c4ea14, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/edc66f64f16341c8aef6894ece7a1345, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f1171d098ab24b2da7b5568f35bd16d2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/9028ba478a754d3fa4dab192edd60a6e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/cba9402b7d3242ae8ac02ed5e54373ba, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f76b82a141c2440f9c65c0de93408835, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/02e2f6385b8b40f085485649df8d22b1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/96d4d059b0184685858e5ff581542752, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/a0ee6ee7aec946a588a39bcceaf9c085, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/15b683fe38af4ccd9aebc4b23c69617f] to archive 2024-11-16T14:16:12,455 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:16:12,456 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/6a046fe006654047b8be25661a395073 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/6a046fe006654047b8be25661a395073 2024-11-16T14:16:12,457 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/02d3c7f7d0bc4df59cfcd1fec042cd8c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/02d3c7f7d0bc4df59cfcd1fec042cd8c 2024-11-16T14:16:12,458 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f93274cec41c4079b5dc8564a6f0c161 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f93274cec41c4079b5dc8564a6f0c161 2024-11-16T14:16:12,459 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/8ef0fb9d4fdc4d638c537d003c845bc4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/8ef0fb9d4fdc4d638c537d003c845bc4 2024-11-16T14:16:12,461 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f7eff9b39a494b5ca6ecdd9eda6f22ba to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f7eff9b39a494b5ca6ecdd9eda6f22ba 2024-11-16T14:16:12,462 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/1c714d99a2bd49ac871f3202a1718302 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/1c714d99a2bd49ac871f3202a1718302 2024-11-16T14:16:12,463 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/ed75ad9ce1ef404c98cabeb617ebf49a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/ed75ad9ce1ef404c98cabeb617ebf49a 2024-11-16T14:16:12,464 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/9738a9d63f094aa3a8265534709dee1c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/9738a9d63f094aa3a8265534709dee1c 2024-11-16T14:16:12,465 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/c962a1314d3c43d9b701fb54b7c4ea14 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/c962a1314d3c43d9b701fb54b7c4ea14 2024-11-16T14:16:12,466 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/edc66f64f16341c8aef6894ece7a1345 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/edc66f64f16341c8aef6894ece7a1345 2024-11-16T14:16:12,467 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f1171d098ab24b2da7b5568f35bd16d2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f1171d098ab24b2da7b5568f35bd16d2 2024-11-16T14:16:12,468 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/9028ba478a754d3fa4dab192edd60a6e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/9028ba478a754d3fa4dab192edd60a6e 2024-11-16T14:16:12,469 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/cba9402b7d3242ae8ac02ed5e54373ba to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/cba9402b7d3242ae8ac02ed5e54373ba 2024-11-16T14:16:12,471 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f76b82a141c2440f9c65c0de93408835 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/f76b82a141c2440f9c65c0de93408835 2024-11-16T14:16:12,472 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/02e2f6385b8b40f085485649df8d22b1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/02e2f6385b8b40f085485649df8d22b1 2024-11-16T14:16:12,473 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/96d4d059b0184685858e5ff581542752 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/96d4d059b0184685858e5ff581542752 2024-11-16T14:16:12,474 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/a0ee6ee7aec946a588a39bcceaf9c085 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/a0ee6ee7aec946a588a39bcceaf9c085 2024-11-16T14:16:12,474 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/15b683fe38af4ccd9aebc4b23c69617f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/15b683fe38af4ccd9aebc4b23c69617f 2024-11-16T14:16:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:16:12,736 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/e9c96d2aaa77483cb908b2a02b1d3f4b 2024-11-16T14:16:12,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/14d9c17406fc460b81a76fc767eed4a7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/14d9c17406fc460b81a76fc767eed4a7 2024-11-16T14:16:12,742 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/14d9c17406fc460b81a76fc767eed4a7, entries=150, sequenceid=272, filesize=30.5 K 2024-11-16T14:16:12,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/bed3318d6b5c4599a957c285e7bf3314 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/bed3318d6b5c4599a957c285e7bf3314 2024-11-16T14:16:12,746 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/bed3318d6b5c4599a957c285e7bf3314, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T14:16:12,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/e9c96d2aaa77483cb908b2a02b1d3f4b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/e9c96d2aaa77483cb908b2a02b1d3f4b 2024-11-16T14:16:12,751 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/e9c96d2aaa77483cb908b2a02b1d3f4b, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T14:16:12,752 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1261ms, sequenceid=272, compaction requested=false 2024-11-16T14:16:12,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:12,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:12,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-16T14:16:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-16T14:16:12,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-16T14:16:12,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4150 sec 2024-11-16T14:16:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:12,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T14:16:12,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4200 sec 2024-11-16T14:16:12,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:16:12,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:12,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:16:12,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:12,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:16:12,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:12,760 INFO [Thread-1543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1543. 2024-11-16T14:16:12,760 DEBUG [Thread-1543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:12,760 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:12,760 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:12,760 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:12,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111648c5c438ac88402b879658cb6209fb72_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766572116/Put/seqid=0 2024-11-16T14:16:12,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742241_1417 (size=12454) 2024-11-16T14:16:13,169 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:13,178 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111648c5c438ac88402b879658cb6209fb72_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111648c5c438ac88402b879658cb6209fb72_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:13,179 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/1eeeb9fb2924456e82fd2905de97f417, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:13,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/1eeeb9fb2924456e82fd2905de97f417 is 175, key is test_row_0/A:col10/1731766572116/Put/seqid=0 2024-11-16T14:16:13,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742242_1418 (size=31255) 2024-11-16T14:16:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:16:13,486 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:13,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/1eeeb9fb2924456e82fd2905de97f417 2024-11-16T14:16:13,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/4b249f9a5aca4a09afe5594fb9046f24 is 50, key is test_row_0/B:col10/1731766572116/Put/seqid=0 2024-11-16T14:16:13,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742243_1419 (size=12301) 2024-11-16T14:16:14,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/4b249f9a5aca4a09afe5594fb9046f24 2024-11-16T14:16:14,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/2aa820422ffa41a0ba0c67ab7526d129 is 50, key is test_row_0/C:col10/1731766572116/Put/seqid=0 2024-11-16T14:16:14,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742244_1420 (size=12301) 2024-11-16T14:16:14,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/2aa820422ffa41a0ba0c67ab7526d129 2024-11-16T14:16:14,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/1eeeb9fb2924456e82fd2905de97f417 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1eeeb9fb2924456e82fd2905de97f417 2024-11-16T14:16:14,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1eeeb9fb2924456e82fd2905de97f417, entries=150, sequenceid=294, filesize=30.5 K 2024-11-16T14:16:14,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/4b249f9a5aca4a09afe5594fb9046f24 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/4b249f9a5aca4a09afe5594fb9046f24 2024-11-16T14:16:14,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/4b249f9a5aca4a09afe5594fb9046f24, entries=150, sequenceid=294, filesize=12.0 K 2024-11-16T14:16:14,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/2aa820422ffa41a0ba0c67ab7526d129 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/2aa820422ffa41a0ba0c67ab7526d129 2024-11-16T14:16:14,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/2aa820422ffa41a0ba0c67ab7526d129, entries=150, sequenceid=294, filesize=12.0 K 2024-11-16T14:16:14,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=0 B/0 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1725ms, sequenceid=294, compaction requested=true 2024-11-16T14:16:14,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:14,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:14,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:14,483 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:14,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:14,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:14,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f43c903d1ca08ce04b9ee6d4b3aa8643:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:14,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:14,483 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:14,483 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:14,483 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:14,484 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/B is initiating minor compaction (all files) 2024-11-16T14:16:14,484 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/A is initiating minor compaction (all files) 2024-11-16T14:16:14,484 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/B in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:14,484 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/A in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:14,484 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/1313e91266fc4d19ae68fb7ea6556a58, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/bed3318d6b5c4599a957c285e7bf3314, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/4b249f9a5aca4a09afe5594fb9046f24] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=36.5 K 2024-11-16T14:16:14,484 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/f4f982275fd244e6b864ac2fe75dddd2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/14d9c17406fc460b81a76fc767eed4a7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1eeeb9fb2924456e82fd2905de97f417] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=92.1 K 2024-11-16T14:16:14,484 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:14,484 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/f4f982275fd244e6b864ac2fe75dddd2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/14d9c17406fc460b81a76fc767eed4a7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1eeeb9fb2924456e82fd2905de97f417] 2024-11-16T14:16:14,484 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1313e91266fc4d19ae68fb7ea6556a58, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731766568564 2024-11-16T14:16:14,484 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting bed3318d6b5c4599a957c285e7bf3314, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731766569800 2024-11-16T14:16:14,484 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4f982275fd244e6b864ac2fe75dddd2, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731766568564 2024-11-16T14:16:14,484 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b249f9a5aca4a09afe5594fb9046f24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731766572088 2024-11-16T14:16:14,484 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14d9c17406fc460b81a76fc767eed4a7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731766569800 2024-11-16T14:16:14,485 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eeeb9fb2924456e82fd2905de97f417, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731766572088 2024-11-16T14:16:14,490 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:14,491 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:14,492 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/b1bbe14f62024e67bc5793805f860ee5 is 50, key is test_row_0/B:col10/1731766572116/Put/seqid=0 2024-11-16T14:16:14,492 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111639bd2f856abd4378aa02528ad1399def_f43c903d1ca08ce04b9ee6d4b3aa8643 store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:14,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742245_1421 (size=12404) 2024-11-16T14:16:14,504 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111639bd2f856abd4378aa02528ad1399def_f43c903d1ca08ce04b9ee6d4b3aa8643, store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:14,504 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111639bd2f856abd4378aa02528ad1399def_f43c903d1ca08ce04b9ee6d4b3aa8643 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:14,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742246_1422 (size=4469) 2024-11-16T14:16:14,902 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/b1bbe14f62024e67bc5793805f860ee5 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/b1bbe14f62024e67bc5793805f860ee5 2024-11-16T14:16:14,908 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#A#compaction#355 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:14,909 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/B of f43c903d1ca08ce04b9ee6d4b3aa8643 into b1bbe14f62024e67bc5793805f860ee5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:14,909 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:14,909 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/B, priority=13, startTime=1731766574483; duration=0sec 2024-11-16T14:16:14,909 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:14,909 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:B 2024-11-16T14:16:14,909 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:14,909 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/26bac393696c4210872a594fcb40d7b3 is 175, key is test_row_0/A:col10/1731766572116/Put/seqid=0 2024-11-16T14:16:14,911 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:14,911 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): f43c903d1ca08ce04b9ee6d4b3aa8643/C is initiating minor compaction (all files) 2024-11-16T14:16:14,911 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f43c903d1ca08ce04b9ee6d4b3aa8643/C in TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:14,911 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/ade6a2890e4c4f9db9eaeef97e068a6c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/e9c96d2aaa77483cb908b2a02b1d3f4b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/2aa820422ffa41a0ba0c67ab7526d129] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp, totalSize=36.5 K 2024-11-16T14:16:14,911 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ade6a2890e4c4f9db9eaeef97e068a6c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731766568564 2024-11-16T14:16:14,912 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting e9c96d2aaa77483cb908b2a02b1d3f4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731766569800 2024-11-16T14:16:14,912 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa820422ffa41a0ba0c67ab7526d129, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731766572088 2024-11-16T14:16:14,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742247_1423 (size=31358) 2024-11-16T14:16:14,924 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f43c903d1ca08ce04b9ee6d4b3aa8643#C#compaction#356 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:14,924 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/b90a2eb6a265401a89188102a904b166 is 50, key is test_row_0/C:col10/1731766572116/Put/seqid=0 2024-11-16T14:16:14,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742248_1424 (size=12404) 2024-11-16T14:16:14,947 INFO [Thread-1541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1541. 2024-11-16T14:16:14,947 DEBUG [Thread-1541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:14,947 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:14,947 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:14,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:14,967 INFO [Thread-1545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1545. 2024-11-16T14:16:14,967 DEBUG [Thread-1545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:14,967 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:14,967 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:14,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:14,997 INFO [Thread-1539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1539. 2024-11-16T14:16:14,997 DEBUG [Thread-1539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:14,997 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:14,998 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:14,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:15,009 INFO [Thread-1537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1537. 2024-11-16T14:16:15,009 DEBUG [Thread-1537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:15,009 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:15,009 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:15,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1079 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3236 rows 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1083 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3248 rows 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1081 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3242 rows 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1089 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3260 rows 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1087 2024-11-16T14:16:15,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3257 rows 2024-11-16T14:16:15,010 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:16:15,011 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:16:15,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:15,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:15,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:15,012 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T14:16:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T14:16:15,017 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766575017"}]},"ts":"1731766575017"} 2024-11-16T14:16:15,019 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T14:16:15,019 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T14:16:15,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:16:15,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, UNASSIGN}] 2024-11-16T14:16:15,022 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, UNASSIGN 2024-11-16T14:16:15,023 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=f43c903d1ca08ce04b9ee6d4b3aa8643, regionState=CLOSING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:15,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, UNASSIGN because future has completed 2024-11-16T14:16:15,025 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:16:15,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:16:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T14:16:15,178 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:15,178 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:16:15,179 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing f43c903d1ca08ce04b9ee6d4b3aa8643, disabling compactions & flushes 2024-11-16T14:16:15,179 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:15,321 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/26bac393696c4210872a594fcb40d7b3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/26bac393696c4210872a594fcb40d7b3 2024-11-16T14:16:15,326 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/A of f43c903d1ca08ce04b9ee6d4b3aa8643 into 26bac393696c4210872a594fcb40d7b3(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:15,326 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:15,326 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/A, priority=13, startTime=1731766574483; duration=0sec 2024-11-16T14:16:15,326 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:15,326 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:A 2024-11-16T14:16:15,332 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/b90a2eb6a265401a89188102a904b166 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/b90a2eb6a265401a89188102a904b166 2024-11-16T14:16:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T14:16:15,336 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f43c903d1ca08ce04b9ee6d4b3aa8643/C of f43c903d1ca08ce04b9ee6d4b3aa8643 into b90a2eb6a265401a89188102a904b166(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:15,336 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f43c903d1ca08ce04b9ee6d4b3aa8643: 2024-11-16T14:16:15,336 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., storeName=f43c903d1ca08ce04b9ee6d4b3aa8643/C, priority=13, startTime=1731766574483; duration=0sec 2024-11-16T14:16:15,336 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:15,336 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:15,336 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:15,336 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f43c903d1ca08ce04b9ee6d4b3aa8643:C 2024-11-16T14:16:15,336 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. after waiting 0 ms 2024-11-16T14:16:15,336 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:15,336 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing f43c903d1ca08ce04b9ee6d4b3aa8643 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-16T14:16:15,337 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=A 2024-11-16T14:16:15,337 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:15,337 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=B 2024-11-16T14:16:15,337 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:15,337 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f43c903d1ca08ce04b9ee6d4b3aa8643, store=C 2024-11-16T14:16:15,337 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:15,342 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d4c0d051973a49ebbcfb6a6dc50d4d41_f43c903d1ca08ce04b9ee6d4b3aa8643 is 50, key is test_row_0/A:col10/1731766574966/Put/seqid=0 2024-11-16T14:16:15,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742249_1425 (size=9914) 2024-11-16T14:16:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T14:16:15,747 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:15,753 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d4c0d051973a49ebbcfb6a6dc50d4d41_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d4c0d051973a49ebbcfb6a6dc50d4d41_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:15,755 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/7ac83d1b31534583abb2a02d486e9e56, store: [table=TestAcidGuarantees family=A region=f43c903d1ca08ce04b9ee6d4b3aa8643] 2024-11-16T14:16:15,756 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/7ac83d1b31534583abb2a02d486e9e56 is 175, key is test_row_0/A:col10/1731766574966/Put/seqid=0 2024-11-16T14:16:15,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742250_1426 (size=22561) 2024-11-16T14:16:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T14:16:16,161 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/7ac83d1b31534583abb2a02d486e9e56 2024-11-16T14:16:16,166 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/d408b3c5ab8b46a0a6699c93f3f62f08 is 50, key is test_row_0/B:col10/1731766574966/Put/seqid=0 2024-11-16T14:16:16,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742251_1427 (size=9857) 2024-11-16T14:16:16,570 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/d408b3c5ab8b46a0a6699c93f3f62f08 2024-11-16T14:16:16,576 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/962bc0fb490546fba5dc71f6f34180e8 is 50, key is test_row_0/C:col10/1731766574966/Put/seqid=0 2024-11-16T14:16:16,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742252_1428 (size=9857) 2024-11-16T14:16:16,980 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/962bc0fb490546fba5dc71f6f34180e8 2024-11-16T14:16:16,983 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/A/7ac83d1b31534583abb2a02d486e9e56 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/7ac83d1b31534583abb2a02d486e9e56 2024-11-16T14:16:16,987 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/7ac83d1b31534583abb2a02d486e9e56, entries=100, sequenceid=304, filesize=22.0 K 2024-11-16T14:16:16,987 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/B/d408b3c5ab8b46a0a6699c93f3f62f08 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/d408b3c5ab8b46a0a6699c93f3f62f08 2024-11-16T14:16:16,990 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/d408b3c5ab8b46a0a6699c93f3f62f08, entries=100, sequenceid=304, filesize=9.6 K 2024-11-16T14:16:16,990 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/.tmp/C/962bc0fb490546fba5dc71f6f34180e8 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/962bc0fb490546fba5dc71f6f34180e8 2024-11-16T14:16:16,993 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/962bc0fb490546fba5dc71f6f34180e8, entries=100, sequenceid=304, filesize=9.6 K 2024-11-16T14:16:16,994 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1658ms, sequenceid=304, compaction requested=false 2024-11-16T14:16:16,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/f4f982275fd244e6b864ac2fe75dddd2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/14d9c17406fc460b81a76fc767eed4a7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1eeeb9fb2924456e82fd2905de97f417] to archive 2024-11-16T14:16:16,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:16:16,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/f4f982275fd244e6b864ac2fe75dddd2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/f4f982275fd244e6b864ac2fe75dddd2 2024-11-16T14:16:16,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/14d9c17406fc460b81a76fc767eed4a7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/14d9c17406fc460b81a76fc767eed4a7 2024-11-16T14:16:16,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1eeeb9fb2924456e82fd2905de97f417 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/1eeeb9fb2924456e82fd2905de97f417 2024-11-16T14:16:16,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/1313e91266fc4d19ae68fb7ea6556a58, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/bed3318d6b5c4599a957c285e7bf3314, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/4b249f9a5aca4a09afe5594fb9046f24] to archive 2024-11-16T14:16:16,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:16:17,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/1313e91266fc4d19ae68fb7ea6556a58 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/1313e91266fc4d19ae68fb7ea6556a58 2024-11-16T14:16:17,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/bed3318d6b5c4599a957c285e7bf3314 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/bed3318d6b5c4599a957c285e7bf3314 2024-11-16T14:16:17,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/4b249f9a5aca4a09afe5594fb9046f24 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/4b249f9a5aca4a09afe5594fb9046f24 2024-11-16T14:16:17,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/ade6a2890e4c4f9db9eaeef97e068a6c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/e9c96d2aaa77483cb908b2a02b1d3f4b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/2aa820422ffa41a0ba0c67ab7526d129] to archive 2024-11-16T14:16:17,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:16:17,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/ade6a2890e4c4f9db9eaeef97e068a6c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/ade6a2890e4c4f9db9eaeef97e068a6c 2024-11-16T14:16:17,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/e9c96d2aaa77483cb908b2a02b1d3f4b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/e9c96d2aaa77483cb908b2a02b1d3f4b 2024-11-16T14:16:17,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/2aa820422ffa41a0ba0c67ab7526d129 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/2aa820422ffa41a0ba0c67ab7526d129 2024-11-16T14:16:17,011 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=4 2024-11-16T14:16:17,011 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. 2024-11-16T14:16:17,011 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for f43c903d1ca08ce04b9ee6d4b3aa8643: Waiting for close lock at 1731766575178Running coprocessor pre-close hooks at 1731766575178Disabling compacts and flushes for region at 1731766575178Disabling writes for close at 1731766575336 (+158 ms)Obtaining lock to block concurrent updates at 1731766575336Preparing flush snapshotting stores in f43c903d1ca08ce04b9ee6d4b3aa8643 at 1731766575336Finished memstore snapshotting TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731766575337 (+1 ms)Flushing stores of TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643. at 1731766575337Flushing f43c903d1ca08ce04b9ee6d4b3aa8643/A: creating writer at 1731766575337Flushing mob file f43c903d1ca08ce04b9ee6d4b3aa8643/A: appending metadata at 1731766575342 (+5 ms)Flushing mob file f43c903d1ca08ce04b9ee6d4b3aa8643/A: closing flushed file at 1731766575342Flushing f43c903d1ca08ce04b9ee6d4b3aa8643/A: appending metadata at 1731766575755 (+413 ms)Flushing f43c903d1ca08ce04b9ee6d4b3aa8643/A: closing flushed file at 1731766575755Flushing f43c903d1ca08ce04b9ee6d4b3aa8643/B: creating writer at 1731766576164 (+409 ms)Flushing f43c903d1ca08ce04b9ee6d4b3aa8643/B: appending metadata at 1731766576166 (+2 ms)Flushing f43c903d1ca08ce04b9ee6d4b3aa8643/B: closing flushed file at 1731766576166Flushing f43c903d1ca08ce04b9ee6d4b3aa8643/C: creating writer at 1731766576574 (+408 ms)Flushing f43c903d1ca08ce04b9ee6d4b3aa8643/C: appending metadata at 1731766576576 (+2 ms)Flushing f43c903d1ca08ce04b9ee6d4b3aa8643/C: closing flushed file at 1731766576576Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d5b04d3: reopening flushed file at 1731766576983 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66512cb2: reopening flushed file at 1731766576987 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c570fe5: reopening flushed file at 1731766576990 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f43c903d1ca08ce04b9ee6d4b3aa8643 in 1658ms, sequenceid=304, compaction requested=false at 1731766576994 (+4 ms)Writing region close event to WAL at 1731766577008 (+14 ms)Running coprocessor post-close hooks at 1731766577011 (+3 ms)Closed at 1731766577011 2024-11-16T14:16:17,013 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,013 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=f43c903d1ca08ce04b9ee6d4b3aa8643, regionState=CLOSED 2024-11-16T14:16:17,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:16:17,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-16T14:16:17,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure f43c903d1ca08ce04b9ee6d4b3aa8643, server=04e93f2b5fad,41317,1731766448342 in 1.9900 sec 2024-11-16T14:16:17,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-16T14:16:17,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f43c903d1ca08ce04b9ee6d4b3aa8643, UNASSIGN in 1.9950 sec 2024-11-16T14:16:17,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-16T14:16:17,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9980 sec 2024-11-16T14:16:17,020 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766577020"}]},"ts":"1731766577020"} 2024-11-16T14:16:17,021 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T14:16:17,021 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T14:16:17,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0100 sec 2024-11-16T14:16:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T14:16:17,166 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:17,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T14:16:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:17,171 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T14:16:17,173 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:17,178 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,181 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/recovered.edits] 2024-11-16T14:16:17,185 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/26bac393696c4210872a594fcb40d7b3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/26bac393696c4210872a594fcb40d7b3 2024-11-16T14:16:17,186 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/7ac83d1b31534583abb2a02d486e9e56 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/A/7ac83d1b31534583abb2a02d486e9e56 2024-11-16T14:16:17,189 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/b1bbe14f62024e67bc5793805f860ee5 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/b1bbe14f62024e67bc5793805f860ee5 2024-11-16T14:16:17,191 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/d408b3c5ab8b46a0a6699c93f3f62f08 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/B/d408b3c5ab8b46a0a6699c93f3f62f08 2024-11-16T14:16:17,193 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/962bc0fb490546fba5dc71f6f34180e8 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/962bc0fb490546fba5dc71f6f34180e8 2024-11-16T14:16:17,195 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/b90a2eb6a265401a89188102a904b166 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/C/b90a2eb6a265401a89188102a904b166 2024-11-16T14:16:17,198 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/recovered.edits/307.seqid to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643/recovered.edits/307.seqid 2024-11-16T14:16:17,198 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,198 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T14:16:17,199 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T14:16:17,200 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T14:16:17,203 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160179adb37a1e41f4a20a64d1d9c68bb9_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160179adb37a1e41f4a20a64d1d9c68bb9_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,205 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161e89c3d20a68496ba8721c354eec51a7_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161e89c3d20a68496ba8721c354eec51a7_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,207 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161ed1907f8e9247f8a0a790fd07fd32d1_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161ed1907f8e9247f8a0a790fd07fd32d1_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,208 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636cedc85b1df4ddda1e028e5ce1d97ef_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636cedc85b1df4ddda1e028e5ce1d97ef_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,210 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164345b757e7824f87a15dd96bc9eeaaf5_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164345b757e7824f87a15dd96bc9eeaaf5_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,212 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111648c5c438ac88402b879658cb6209fb72_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111648c5c438ac88402b879658cb6209fb72_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,213 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165c1ccef2e1d04167acbf709a5d492dbd_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165c1ccef2e1d04167acbf709a5d492dbd_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,215 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166328f7f9ece44ebca784f3e499c58d2d_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166328f7f9ece44ebca784f3e499c58d2d_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,217 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111665e2ba33c7c3418ba83dec6d012860c8_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111665e2ba33c7c3418ba83dec6d012860c8_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,219 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116692c4908646a476ab44bdc5c30cba260_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116692c4908646a476ab44bdc5c30cba260_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,220 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168b4c432d347f4bbfb943fa8a2d964f7a_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168b4c432d347f4bbfb943fa8a2d964f7a_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,222 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b08684f5bf434be3a181da0be055379d_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b08684f5bf434be3a181da0be055379d_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,223 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d4c0d051973a49ebbcfb6a6dc50d4d41_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d4c0d051973a49ebbcfb6a6dc50d4d41_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,224 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116db77945620674220896980ace6da006c_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116db77945620674220896980ace6da006c_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,225 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116efaf63df02c24f0bb74a0b2ee35140e1_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116efaf63df02c24f0bb74a0b2ee35140e1_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,226 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f706cef0846c4a90a6572b140d91a503_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f706cef0846c4a90a6572b140d91a503_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,227 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fae3f82fe5474f929029cad3dc075520_f43c903d1ca08ce04b9ee6d4b3aa8643 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fae3f82fe5474f929029cad3dc075520_f43c903d1ca08ce04b9ee6d4b3aa8643 2024-11-16T14:16:17,227 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T14:16:17,229 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:17,231 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T14:16:17,233 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T14:16:17,234 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:17,234 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T14:16:17,234 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731766577234"}]},"ts":"9223372036854775807"} 2024-11-16T14:16:17,236 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T14:16:17,236 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f43c903d1ca08ce04b9ee6d4b3aa8643, NAME => 'TestAcidGuarantees,,1731766549228.f43c903d1ca08ce04b9ee6d4b3aa8643.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T14:16:17,236 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T14:16:17,236 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731766577236"}]},"ts":"9223372036854775807"} 2024-11-16T14:16:17,238 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T14:16:17,238 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:17,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-16T14:16:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T14:16:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T14:16:17,276 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:17,283 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 250) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1474060206_22 at /127.0.0.1:60460 [Waiting for operation #627] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/04e93f2b5fad:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1474060206_22 at /127.0.0.1:57966 [Waiting for operation #975] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-571885164_22 at /127.0.0.1:60494 [Waiting for operation #599] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-571885164_22 at /127.0.0.1:44816 [Waiting for operation #446] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=414 (was 438), ProcessCount=11 (was 11), AvailableMemoryMB=3968 (was 4018) 2024-11-16T14:16:17,290 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=414, ProcessCount=11, AvailableMemoryMB=3968 2024-11-16T14:16:17,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:16:17,292 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:16:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:17,294 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:16:17,294 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:17,294 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-16T14:16:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-16T14:16:17,294 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:16:17,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742253_1429 (size=1061) 2024-11-16T14:16:17,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-16T14:16:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-16T14:16:17,702 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc 2024-11-16T14:16:17,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742254_1430 (size=53) 2024-11-16T14:16:17,873 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T14:16:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-16T14:16:18,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:16:18,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4824a7508eaae82911857430ab75750c, disabling compactions & flushes 2024-11-16T14:16:18,115 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:18,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:18,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. after waiting 0 ms 2024-11-16T14:16:18,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:18,115 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:18,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4824a7508eaae82911857430ab75750c: Waiting for close lock at 1731766578115Disabling compacts and flushes for region at 1731766578115Disabling writes for close at 1731766578115Writing region close event to WAL at 1731766578115Closed at 1731766578115 2024-11-16T14:16:18,119 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:16:18,119 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731766578119"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731766578119"}]},"ts":"1731766578119"} 2024-11-16T14:16:18,122 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:16:18,123 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:16:18,124 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766578123"}]},"ts":"1731766578123"} 2024-11-16T14:16:18,126 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T14:16:18,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4824a7508eaae82911857430ab75750c, ASSIGN}] 2024-11-16T14:16:18,127 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4824a7508eaae82911857430ab75750c, ASSIGN 2024-11-16T14:16:18,128 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4824a7508eaae82911857430ab75750c, ASSIGN; state=OFFLINE, location=04e93f2b5fad,41317,1731766448342; forceNewPlan=false, retain=false 2024-11-16T14:16:18,279 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=4824a7508eaae82911857430ab75750c, regionState=OPENING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:18,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4824a7508eaae82911857430ab75750c, ASSIGN because future has completed 2024-11-16T14:16:18,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:16:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-16T14:16:18,440 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:18,440 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:16:18,441 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,441 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:16:18,441 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,441 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,443 INFO [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,444 INFO [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:16:18,444 INFO [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4824a7508eaae82911857430ab75750c columnFamilyName A 2024-11-16T14:16:18,444 DEBUG [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:18,444 INFO [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] regionserver.HStore(327): Store=4824a7508eaae82911857430ab75750c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:16:18,444 INFO [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,445 INFO [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:16:18,445 INFO [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4824a7508eaae82911857430ab75750c columnFamilyName B 2024-11-16T14:16:18,445 DEBUG [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:18,446 INFO [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] regionserver.HStore(327): Store=4824a7508eaae82911857430ab75750c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:16:18,446 INFO [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,447 INFO [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:16:18,447 INFO [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4824a7508eaae82911857430ab75750c columnFamilyName C 2024-11-16T14:16:18,447 DEBUG [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:18,447 INFO [StoreOpener-4824a7508eaae82911857430ab75750c-1 {}] regionserver.HStore(327): Store=4824a7508eaae82911857430ab75750c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:16:18,447 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:18,447 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,448 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,448 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,449 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,450 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,450 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:16:18,451 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,453 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:16:18,454 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 4824a7508eaae82911857430ab75750c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67953600, jitterRate=0.012587547302246094}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:16:18,454 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:18,455 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 4824a7508eaae82911857430ab75750c: Running coprocessor pre-open hook at 1731766578441Writing region info on filesystem at 1731766578441Initializing all the Stores at 1731766578442 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766578442Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766578442Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766578442Cleaning up temporary data from old regions at 1731766578450 (+8 ms)Running coprocessor post-open hooks at 1731766578454 (+4 ms)Region opened successfully at 1731766578455 (+1 ms) 2024-11-16T14:16:18,456 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., pid=116, masterSystemTime=1731766578438 2024-11-16T14:16:18,458 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:18,458 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:18,459 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=4824a7508eaae82911857430ab75750c, regionState=OPEN, openSeqNum=2, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:18,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:16:18,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-16T14:16:18,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 in 178 msec 2024-11-16T14:16:18,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-16T14:16:18,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4824a7508eaae82911857430ab75750c, ASSIGN in 337 msec 2024-11-16T14:16:18,466 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:16:18,466 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766578466"}]},"ts":"1731766578466"} 2024-11-16T14:16:18,468 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T14:16:18,469 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:16:18,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-11-16T14:16:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-16T14:16:19,446 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:19,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ad78a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,448 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:19,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:19,449 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13a4cc3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:19,453 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d5481b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,454 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:19,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:19,457 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:16:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 04e93f2b5fad,39231,1731766447537 2024-11-16T14:16:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6cd29660 2024-11-16T14:16:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:16:19,458 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:16:19,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:19,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:19,460 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581809b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:19,461 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66308fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,462 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:19,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:19,462 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@273d4c2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:19,464 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:19,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c24ce4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:19,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:19,466 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:19,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:19,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:19,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cda7f91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:19,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:19,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:19,467 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:19,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@639398c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,467 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:19,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:19,468 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cec331d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:19,469 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ef1cd20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:19,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:19,471 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65314887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:19,472 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d5b8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:19,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:19,474 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b6b95ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:19,475 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677f3577, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:19,476 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:19,477 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4716db31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:19,478 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25966e7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:19,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:19,479 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13cc263b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:19,481 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679c1588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:19,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:19,482 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@436322aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:19,483 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:19,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3533b527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,484 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:19,484 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:19,485 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63511d2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:19,486 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1efcaa7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:19,487 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:19,487 DEBUG [Thread-1949 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:19,487 DEBUG [Thread-1956 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:19,488 DEBUG [Thread-1951 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:19,488 DEBUG [Thread-1962 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:19,488 DEBUG [Thread-1958 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:19,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:19,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:19,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:19,489 DEBUG [Thread-1960 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:19,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:19,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:19,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:19,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:16:19,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:16:19,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:16:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:16:19,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:16:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:16:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:16:19,496 DEBUG [Thread-1943 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:19,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:19,499 DEBUG [Thread-1947 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:16:19,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:19,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:19,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:19,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:16:19,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:19,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:19,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:19,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:19,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:19,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:19,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:16:19,505 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-16T14:16:19,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:19,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:19,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T14:16:19,516 DEBUG [Thread-1954 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:19,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2] 2024-11-16T14:16:19,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731766579519, max timestamp=1731766579530Num uniques 859; 2024-11-16T14:16:19,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:16:19,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731766579519, max timestamp=1731766579530Num uniques 859; 2024-11-16T14:16:19,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:16:19,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766579519, max timestamp=1731766579530Num uniques 859; 2024-11-16T14:16:19,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:16:19,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/e91b095a689d48f9b02d3ffdcb683b17 is 50, key is test_row_0/A:col10/1731766579499/Put/seqid=0 2024-11-16T14:16:19,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742255_1431 (size=16681) 2024-11-16T14:16:19,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/e91b095a689d48f9b02d3ffdcb683b17 2024-11-16T14:16:19,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766639561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766639570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/944aa40ae8c74b14bae0babebe1fd157 is 50, key is test_row_0/B:col10/1731766579499/Put/seqid=0 2024-11-16T14:16:19,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742256_1432 (size=12001) 2024-11-16T14:16:19,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766639571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39412 deadline: 1731766639572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766639572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T14:16:19,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T14:16:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:19,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:19,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766639675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766639685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766639686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766639696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39412 deadline: 1731766639696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T14:16:19,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:19,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:19,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:19,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:19,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:19,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T14:16:19,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766639885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766639896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766639896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766639915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39412 deadline: 1731766639915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:19,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T14:16:19,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:19,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:19,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:19,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:19,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:19,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/944aa40ae8c74b14bae0babebe1fd157 2024-11-16T14:16:20,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/196adfa8c7884ebd908987600bcc21a3 is 50, key is test_row_0/C:col10/1731766579499/Put/seqid=0 2024-11-16T14:16:20,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742257_1433 (size=12001) 2024-11-16T14:16:20,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/196adfa8c7884ebd908987600bcc21a3 2024-11-16T14:16:20,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/e91b095a689d48f9b02d3ffdcb683b17 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/e91b095a689d48f9b02d3ffdcb683b17 2024-11-16T14:16:20,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/e91b095a689d48f9b02d3ffdcb683b17, entries=250, sequenceid=12, filesize=16.3 K 2024-11-16T14:16:20,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/944aa40ae8c74b14bae0babebe1fd157 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/944aa40ae8c74b14bae0babebe1fd157 2024-11-16T14:16:20,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/944aa40ae8c74b14bae0babebe1fd157, entries=150, sequenceid=12, filesize=11.7 K 2024-11-16T14:16:20,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/196adfa8c7884ebd908987600bcc21a3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/196adfa8c7884ebd908987600bcc21a3 2024-11-16T14:16:20,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/196adfa8c7884ebd908987600bcc21a3, entries=150, sequenceid=12, filesize=11.7 K 2024-11-16T14:16:20,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4824a7508eaae82911857430ab75750c in 518ms, sequenceid=12, compaction requested=false 2024-11-16T14:16:20,019 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T14:16:20,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:20,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T14:16:20,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:20,117 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-16T14:16:20,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:20,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/31d34c591f5a4c6fab19932e99413eab is 50, key is test_row_0/A:col10/1731766579568/Put/seqid=0 2024-11-16T14:16:20,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742258_1434 (size=12001) 2024-11-16T14:16:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T14:16:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:20,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:20,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39412 deadline: 1731766640225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766640225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766640226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766640229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766640230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766640335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766640345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766640346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766640356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,524 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/31d34c591f5a4c6fab19932e99413eab 2024-11-16T14:16:20,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/c3b5b0cf475147fca8d36f355dc72053 is 50, key is test_row_0/B:col10/1731766579568/Put/seqid=0 2024-11-16T14:16:20,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742259_1435 (size=12001) 2024-11-16T14:16:20,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766640555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766640555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766640556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766640566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T14:16:20,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39412 deadline: 1731766640735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766640865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766640865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766640866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766640876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:20,934 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/c3b5b0cf475147fca8d36f355dc72053 2024-11-16T14:16:20,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/1d7bcfc8126c4e3bad92e11612bc5b21 is 50, key is test_row_0/C:col10/1731766579568/Put/seqid=0 2024-11-16T14:16:20,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742260_1436 (size=12001) 2024-11-16T14:16:21,349 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/1d7bcfc8126c4e3bad92e11612bc5b21 2024-11-16T14:16:21,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/31d34c591f5a4c6fab19932e99413eab as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/31d34c591f5a4c6fab19932e99413eab 2024-11-16T14:16:21,367 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/31d34c591f5a4c6fab19932e99413eab, entries=150, sequenceid=47, filesize=11.7 K 2024-11-16T14:16:21,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/c3b5b0cf475147fca8d36f355dc72053 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c3b5b0cf475147fca8d36f355dc72053 2024-11-16T14:16:21,372 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c3b5b0cf475147fca8d36f355dc72053, entries=150, sequenceid=47, filesize=11.7 K 2024-11-16T14:16:21,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/1d7bcfc8126c4e3bad92e11612bc5b21 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/1d7bcfc8126c4e3bad92e11612bc5b21 2024-11-16T14:16:21,376 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/1d7bcfc8126c4e3bad92e11612bc5b21, entries=150, sequenceid=47, filesize=11.7 K 2024-11-16T14:16:21,377 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 4824a7508eaae82911857430ab75750c in 1260ms, sequenceid=47, compaction requested=false 2024-11-16T14:16:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-16T14:16:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-16T14:16:21,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-16T14:16:21,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8730 sec 2024-11-16T14:16:21,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8760 sec 2024-11-16T14:16:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:21,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:16:21,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:21,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:21,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:21,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:21,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:21,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:21,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/56fb3866469c4641a166fc893f2eae83 is 50, key is test_row_0/A:col10/1731766580227/Put/seqid=0 2024-11-16T14:16:21,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742261_1437 (size=14341) 2024-11-16T14:16:21,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/56fb3866469c4641a166fc893f2eae83 2024-11-16T14:16:21,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/220ccc654941402e9c948d3977d91959 is 50, key is test_row_0/B:col10/1731766580227/Put/seqid=0 2024-11-16T14:16:21,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742262_1438 (size=12001) 2024-11-16T14:16:21,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/220ccc654941402e9c948d3977d91959 2024-11-16T14:16:21,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/c75124e7654e4c90a491ab3c09241476 is 50, key is test_row_0/C:col10/1731766580227/Put/seqid=0 2024-11-16T14:16:21,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766641432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766641433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766641434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742263_1439 (size=12001) 2024-11-16T14:16:21,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766641449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766641555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766641555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766641555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766641565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T14:16:21,655 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:21,656 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-16T14:16:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T14:16:21,657 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:21,658 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:21,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:21,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39412 deadline: 1731766641746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,753 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in 4824a7508eaae82911857430ab75750c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:21,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T14:16:21,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766641775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766641775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766641776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766641786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:21,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:21,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:21,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T14:16:21,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:21,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:21,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:21,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:21,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:21,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/c75124e7654e4c90a491ab3c09241476 2024-11-16T14:16:21,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/56fb3866469c4641a166fc893f2eae83 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/56fb3866469c4641a166fc893f2eae83 2024-11-16T14:16:21,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/56fb3866469c4641a166fc893f2eae83, entries=200, sequenceid=59, filesize=14.0 K 2024-11-16T14:16:21,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/220ccc654941402e9c948d3977d91959 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/220ccc654941402e9c948d3977d91959 2024-11-16T14:16:21,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/220ccc654941402e9c948d3977d91959, entries=150, sequenceid=59, filesize=11.7 K 2024-11-16T14:16:21,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/c75124e7654e4c90a491ab3c09241476 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c75124e7654e4c90a491ab3c09241476 2024-11-16T14:16:21,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c75124e7654e4c90a491ab3c09241476, entries=150, sequenceid=59, filesize=11.7 K 2024-11-16T14:16:21,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4824a7508eaae82911857430ab75750c in 478ms, sequenceid=59, compaction requested=true 2024-11-16T14:16:21,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:21,868 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:21,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:21,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:21,869 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:21,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:21,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:21,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:21,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:21,872 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:21,872 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/A is initiating minor compaction (all files) 2024-11-16T14:16:21,872 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/A in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:21,872 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:21,872 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/e91b095a689d48f9b02d3ffdcb683b17, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/31d34c591f5a4c6fab19932e99413eab, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/56fb3866469c4641a166fc893f2eae83] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=42.0 K 2024-11-16T14:16:21,872 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/B is initiating minor compaction (all files) 2024-11-16T14:16:21,873 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/B in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:21,873 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/944aa40ae8c74b14bae0babebe1fd157, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c3b5b0cf475147fca8d36f355dc72053, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/220ccc654941402e9c948d3977d91959] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=35.2 K 2024-11-16T14:16:21,873 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e91b095a689d48f9b02d3ffdcb683b17, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731766579493 2024-11-16T14:16:21,873 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 944aa40ae8c74b14bae0babebe1fd157, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731766579493 2024-11-16T14:16:21,873 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31d34c591f5a4c6fab19932e99413eab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731766579556 2024-11-16T14:16:21,873 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b5b0cf475147fca8d36f355dc72053, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731766579556 2024-11-16T14:16:21,873 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56fb3866469c4641a166fc893f2eae83, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731766580213 2024-11-16T14:16:21,873 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 220ccc654941402e9c948d3977d91959, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731766580213 2024-11-16T14:16:21,878 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:21,878 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/781459e5c129415684ac40aeb3adf855 is 50, key is test_row_0/B:col10/1731766580227/Put/seqid=0 2024-11-16T14:16:21,879 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#A#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:21,879 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/f7f99ad32949429ab204d61cd69bd9b0 is 50, key is test_row_0/A:col10/1731766580227/Put/seqid=0 2024-11-16T14:16:21,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742264_1440 (size=12104) 2024-11-16T14:16:21,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742265_1441 (size=12104) 2024-11-16T14:16:21,894 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/f7f99ad32949429ab204d61cd69bd9b0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/f7f99ad32949429ab204d61cd69bd9b0 2024-11-16T14:16:21,898 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/A of 4824a7508eaae82911857430ab75750c into f7f99ad32949429ab204d61cd69bd9b0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:21,898 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:21,898 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/A, priority=13, startTime=1731766581868; duration=0sec 2024-11-16T14:16:21,898 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:21,898 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:A 2024-11-16T14:16:21,898 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:21,898 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:21,898 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/C is initiating minor compaction (all files) 2024-11-16T14:16:21,898 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/C in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:21,899 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/196adfa8c7884ebd908987600bcc21a3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/1d7bcfc8126c4e3bad92e11612bc5b21, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c75124e7654e4c90a491ab3c09241476] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=35.2 K 2024-11-16T14:16:21,899 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 196adfa8c7884ebd908987600bcc21a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731766579493 2024-11-16T14:16:21,899 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d7bcfc8126c4e3bad92e11612bc5b21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731766579556 2024-11-16T14:16:21,899 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c75124e7654e4c90a491ab3c09241476, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731766580213 2024-11-16T14:16:21,905 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:21,905 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/9419a1ff6df647ec87f18ffaf689c43c is 50, key is test_row_0/C:col10/1731766580227/Put/seqid=0 2024-11-16T14:16:21,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742266_1442 (size=12104) 2024-11-16T14:16:21,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T14:16:21,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:21,962 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:16:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/bce8c8f75ce44c26a7f70dc19c9adad4 is 50, key is test_row_0/A:col10/1731766581428/Put/seqid=0 2024-11-16T14:16:21,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742267_1443 (size=12001) 2024-11-16T14:16:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T14:16:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:22,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:22,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766642140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766642141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766642142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766642143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766642245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766642256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766642266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766642266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T14:16:22,292 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/781459e5c129415684ac40aeb3adf855 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/781459e5c129415684ac40aeb3adf855 2024-11-16T14:16:22,296 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/B of 4824a7508eaae82911857430ab75750c into 781459e5c129415684ac40aeb3adf855(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:22,296 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:22,296 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/B, priority=13, startTime=1731766581869; duration=0sec 2024-11-16T14:16:22,296 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:22,296 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:B 2024-11-16T14:16:22,312 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/9419a1ff6df647ec87f18ffaf689c43c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/9419a1ff6df647ec87f18ffaf689c43c 2024-11-16T14:16:22,316 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/C of 4824a7508eaae82911857430ab75750c into 9419a1ff6df647ec87f18ffaf689c43c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:22,316 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:22,316 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/C, priority=13, startTime=1731766581869; duration=0sec 2024-11-16T14:16:22,316 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:22,316 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:C 2024-11-16T14:16:22,371 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/bce8c8f75ce44c26a7f70dc19c9adad4 2024-11-16T14:16:22,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/1c7f8b5e31f041759eceecef004632be is 50, key is test_row_0/B:col10/1731766581428/Put/seqid=0 2024-11-16T14:16:22,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742268_1444 (size=12001) 2024-11-16T14:16:22,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766642456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766642466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766642475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766642487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,527 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:16:22,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766642776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,787 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/1c7f8b5e31f041759eceecef004632be 2024-11-16T14:16:22,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766642786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/a93cdc38fafb4572b3e656ae9878bb8c is 50, key is test_row_0/C:col10/1731766581428/Put/seqid=0 2024-11-16T14:16:22,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742269_1445 (size=12001) 2024-11-16T14:16:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T14:16:22,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766642796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:22,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766642806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:22,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,196 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/a93cdc38fafb4572b3e656ae9878bb8c 2024-11-16T14:16:23,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/bce8c8f75ce44c26a7f70dc19c9adad4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/bce8c8f75ce44c26a7f70dc19c9adad4 2024-11-16T14:16:23,202 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/bce8c8f75ce44c26a7f70dc19c9adad4, entries=150, sequenceid=80, filesize=11.7 K 2024-11-16T14:16:23,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/1c7f8b5e31f041759eceecef004632be as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/1c7f8b5e31f041759eceecef004632be 2024-11-16T14:16:23,206 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/1c7f8b5e31f041759eceecef004632be, entries=150, sequenceid=80, filesize=11.7 K 2024-11-16T14:16:23,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/a93cdc38fafb4572b3e656ae9878bb8c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/a93cdc38fafb4572b3e656ae9878bb8c 2024-11-16T14:16:23,210 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/a93cdc38fafb4572b3e656ae9878bb8c, entries=150, sequenceid=80, filesize=11.7 K 2024-11-16T14:16:23,210 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4824a7508eaae82911857430ab75750c in 1248ms, sequenceid=80, compaction requested=false 2024-11-16T14:16:23,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:23,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:23,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-16T14:16:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-16T14:16:23,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-16T14:16:23,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5540 sec 2024-11-16T14:16:23,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.5580 sec 2024-11-16T14:16:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:23,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:16:23,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:23,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:23,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:23,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:23,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:23,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:23,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/dc69acb3f8744c9fbf0368d1b00b2cf7 is 50, key is test_row_0/A:col10/1731766582110/Put/seqid=0 2024-11-16T14:16:23,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742270_1446 (size=14341) 2024-11-16T14:16:23,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766643325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766643326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766643330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766643331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766643436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766643445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766643445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766643445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766643656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766643656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766643656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766643656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/dc69acb3f8744c9fbf0368d1b00b2cf7 2024-11-16T14:16:23,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/35bfe85b4713483281aa8890222cc94f is 50, key is test_row_0/B:col10/1731766582110/Put/seqid=0 2024-11-16T14:16:23,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742271_1447 (size=12001) 2024-11-16T14:16:23,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/35bfe85b4713483281aa8890222cc94f 2024-11-16T14:16:23,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/8aa42a0a48b944c498cb99922c1de37a is 50, key is test_row_0/C:col10/1731766582110/Put/seqid=0 2024-11-16T14:16:23,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742272_1448 (size=12001) 2024-11-16T14:16:23,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39412 deadline: 1731766643765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,775 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in 4824a7508eaae82911857430ab75750c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T14:16:23,806 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:23,806 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-16T14:16:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T14:16:23,808 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:23,808 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:23,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T14:16:23,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T14:16:23,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:23,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:23,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:23,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:23,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766643966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766643966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766643966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:23,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766643975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T14:16:24,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:24,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:24,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:24,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:24,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T14:16:24,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/8aa42a0a48b944c498cb99922c1de37a 2024-11-16T14:16:24,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/dc69acb3f8744c9fbf0368d1b00b2cf7 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/dc69acb3f8744c9fbf0368d1b00b2cf7 2024-11-16T14:16:24,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/dc69acb3f8744c9fbf0368d1b00b2cf7, entries=200, sequenceid=95, filesize=14.0 K 2024-11-16T14:16:24,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/35bfe85b4713483281aa8890222cc94f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/35bfe85b4713483281aa8890222cc94f 2024-11-16T14:16:24,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/35bfe85b4713483281aa8890222cc94f, entries=150, sequenceid=95, filesize=11.7 K 2024-11-16T14:16:24,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/8aa42a0a48b944c498cb99922c1de37a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/8aa42a0a48b944c498cb99922c1de37a 2024-11-16T14:16:24,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/8aa42a0a48b944c498cb99922c1de37a, entries=150, sequenceid=95, filesize=11.7 K 2024-11-16T14:16:24,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4824a7508eaae82911857430ab75750c in 893ms, sequenceid=95, compaction requested=true 2024-11-16T14:16:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:24,183 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:24,183 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:24,184 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:24,184 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/A is initiating minor compaction (all files) 2024-11-16T14:16:24,184 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:24,184 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/A in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:24,184 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/B is initiating minor compaction (all files) 2024-11-16T14:16:24,184 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/f7f99ad32949429ab204d61cd69bd9b0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/bce8c8f75ce44c26a7f70dc19c9adad4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/dc69acb3f8744c9fbf0368d1b00b2cf7] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=37.5 K 2024-11-16T14:16:24,184 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/B in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:24,184 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/781459e5c129415684ac40aeb3adf855, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/1c7f8b5e31f041759eceecef004632be, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/35bfe85b4713483281aa8890222cc94f] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=35.3 K 2024-11-16T14:16:24,184 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7f99ad32949429ab204d61cd69bd9b0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731766580213 2024-11-16T14:16:24,184 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 781459e5c129415684ac40aeb3adf855, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731766580213 2024-11-16T14:16:24,184 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting bce8c8f75ce44c26a7f70dc19c9adad4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731766581428 2024-11-16T14:16:24,184 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c7f8b5e31f041759eceecef004632be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731766581428 2024-11-16T14:16:24,184 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc69acb3f8744c9fbf0368d1b00b2cf7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731766582110 2024-11-16T14:16:24,185 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 35bfe85b4713483281aa8890222cc94f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731766582110 2024-11-16T14:16:24,195 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#B#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:24,195 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#A#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:24,195 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/72ab0a4694b34f389a41f3998b364dd0 is 50, key is test_row_0/B:col10/1731766582110/Put/seqid=0 2024-11-16T14:16:24,195 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/358d352e476746c1ace1927cbd4bfb46 is 50, key is test_row_0/A:col10/1731766582110/Put/seqid=0 2024-11-16T14:16:24,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742274_1450 (size=12207) 2024-11-16T14:16:24,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742273_1449 (size=12207) 2024-11-16T14:16:24,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T14:16:24,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:24,265 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:16:24,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:24,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:24,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:24,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:24,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:24,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:24,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/f38715c10f984818bce1c74fe60b0942 is 50, key is test_row_0/A:col10/1731766583329/Put/seqid=0 2024-11-16T14:16:24,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742275_1451 (size=12001) 2024-11-16T14:16:24,273 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/f38715c10f984818bce1c74fe60b0942 2024-11-16T14:16:24,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/ca89df5d4370400ab1a21973db3ed27e is 50, key is test_row_0/B:col10/1731766583329/Put/seqid=0 2024-11-16T14:16:24,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742276_1452 (size=12001) 2024-11-16T14:16:24,286 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/ca89df5d4370400ab1a21973db3ed27e 2024-11-16T14:16:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/10c6f0d941fb40548c1ecef1b841c9f1 is 50, key is test_row_0/C:col10/1731766583329/Put/seqid=0 2024-11-16T14:16:24,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742277_1453 (size=12001) 2024-11-16T14:16:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T14:16:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:24,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:24,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766644500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766644502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766644502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766644507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,605 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/358d352e476746c1ace1927cbd4bfb46 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/358d352e476746c1ace1927cbd4bfb46 2024-11-16T14:16:24,605 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/72ab0a4694b34f389a41f3998b364dd0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/72ab0a4694b34f389a41f3998b364dd0 2024-11-16T14:16:24,608 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/B of 4824a7508eaae82911857430ab75750c into 72ab0a4694b34f389a41f3998b364dd0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:24,608 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/A of 4824a7508eaae82911857430ab75750c into 358d352e476746c1ace1927cbd4bfb46(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:24,608 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:24,608 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:24,608 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/B, priority=13, startTime=1731766584183; duration=0sec 2024-11-16T14:16:24,608 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/A, priority=13, startTime=1731766584183; duration=0sec 2024-11-16T14:16:24,608 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:24,608 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:24,608 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:B 2024-11-16T14:16:24,608 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:A 2024-11-16T14:16:24,608 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:24,609 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:24,609 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/C is initiating minor compaction (all files) 2024-11-16T14:16:24,609 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/C in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:24,609 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/9419a1ff6df647ec87f18ffaf689c43c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/a93cdc38fafb4572b3e656ae9878bb8c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/8aa42a0a48b944c498cb99922c1de37a] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=35.3 K 2024-11-16T14:16:24,609 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9419a1ff6df647ec87f18ffaf689c43c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731766580213 2024-11-16T14:16:24,610 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting a93cdc38fafb4572b3e656ae9878bb8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731766581428 2024-11-16T14:16:24,610 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aa42a0a48b944c498cb99922c1de37a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731766582110 2024-11-16T14:16:24,618 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#C#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:24,618 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/956169663db54b80987c279c7d4b7087 is 50, key is test_row_0/C:col10/1731766582110/Put/seqid=0 2024-11-16T14:16:24,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742278_1454 (size=12207) 2024-11-16T14:16:24,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766644616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766644616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766644625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766644625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,697 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/10c6f0d941fb40548c1ecef1b841c9f1 2024-11-16T14:16:24,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/f38715c10f984818bce1c74fe60b0942 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/f38715c10f984818bce1c74fe60b0942 2024-11-16T14:16:24,704 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/f38715c10f984818bce1c74fe60b0942, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T14:16:24,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/ca89df5d4370400ab1a21973db3ed27e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/ca89df5d4370400ab1a21973db3ed27e 2024-11-16T14:16:24,707 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/ca89df5d4370400ab1a21973db3ed27e, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T14:16:24,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/10c6f0d941fb40548c1ecef1b841c9f1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/10c6f0d941fb40548c1ecef1b841c9f1 2024-11-16T14:16:24,710 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/10c6f0d941fb40548c1ecef1b841c9f1, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T14:16:24,711 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4824a7508eaae82911857430ab75750c in 446ms, sequenceid=114, compaction requested=false 2024-11-16T14:16:24,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:24,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:24,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-16T14:16:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-16T14:16:24,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-16T14:16:24,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 905 msec 2024-11-16T14:16:24,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 908 msec 2024-11-16T14:16:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:24,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T14:16:24,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:24,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:24,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:24,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:24,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:24,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:24,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/c44b1f7a40014917999ac872d3607cf0 is 50, key is test_row_0/A:col10/1731766584839/Put/seqid=0 2024-11-16T14:16:24,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742279_1455 (size=14391) 2024-11-16T14:16:24,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766644868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766644871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766644872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766644873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T14:16:24,946 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:24,946 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-16T14:16:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T14:16:24,948 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:24,948 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:24,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:24,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766644975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766644985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766644996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766644996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,024 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/956169663db54b80987c279c7d4b7087 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/956169663db54b80987c279c7d4b7087 2024-11-16T14:16:25,027 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/C of 4824a7508eaae82911857430ab75750c into 956169663db54b80987c279c7d4b7087(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:25,027 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:25,027 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/C, priority=13, startTime=1731766584183; duration=0sec 2024-11-16T14:16:25,028 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:25,028 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:C 2024-11-16T14:16:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T14:16:25,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T14:16:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:25,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:25,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766645195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766645206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766645206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766645206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/c44b1f7a40014917999ac872d3607cf0 2024-11-16T14:16:25,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T14:16:25,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:25,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:25,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:25,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:25,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:25,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/46afb44a3cf745eab4accb8ab318c547 is 50, key is test_row_0/B:col10/1731766584839/Put/seqid=0 2024-11-16T14:16:25,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742280_1456 (size=12051) 2024-11-16T14:16:25,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/46afb44a3cf745eab4accb8ab318c547 2024-11-16T14:16:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T14:16:25,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/cab720838001492bb32cfb3c176c4b2c is 50, key is test_row_0/C:col10/1731766584839/Put/seqid=0 2024-11-16T14:16:25,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742281_1457 (size=12051) 2024-11-16T14:16:25,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/cab720838001492bb32cfb3c176c4b2c 2024-11-16T14:16:25,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/c44b1f7a40014917999ac872d3607cf0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/c44b1f7a40014917999ac872d3607cf0 2024-11-16T14:16:25,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/c44b1f7a40014917999ac872d3607cf0, entries=200, sequenceid=130, filesize=14.1 K 2024-11-16T14:16:25,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/46afb44a3cf745eab4accb8ab318c547 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/46afb44a3cf745eab4accb8ab318c547 2024-11-16T14:16:25,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/46afb44a3cf745eab4accb8ab318c547, entries=150, sequenceid=130, filesize=11.8 K 2024-11-16T14:16:25,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/cab720838001492bb32cfb3c176c4b2c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/cab720838001492bb32cfb3c176c4b2c 2024-11-16T14:16:25,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/cab720838001492bb32cfb3c176c4b2c, entries=150, sequenceid=130, filesize=11.8 K 2024-11-16T14:16:25,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4824a7508eaae82911857430ab75750c in 453ms, sequenceid=130, compaction requested=true 2024-11-16T14:16:25,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:25,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:25,293 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:25,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:25,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:25,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:25,294 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:25,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:25,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:25,294 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:25,294 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:25,294 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/B is initiating minor compaction (all files) 2024-11-16T14:16:25,294 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/A is initiating minor compaction (all files) 2024-11-16T14:16:25,294 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/A in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:25,294 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/B in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:25,294 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/72ab0a4694b34f389a41f3998b364dd0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/ca89df5d4370400ab1a21973db3ed27e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/46afb44a3cf745eab4accb8ab318c547] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=35.4 K 2024-11-16T14:16:25,294 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/358d352e476746c1ace1927cbd4bfb46, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/f38715c10f984818bce1c74fe60b0942, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/c44b1f7a40014917999ac872d3607cf0] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=37.7 K 2024-11-16T14:16:25,295 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ab0a4694b34f389a41f3998b364dd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731766582110 2024-11-16T14:16:25,295 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 358d352e476746c1ace1927cbd4bfb46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731766582110 2024-11-16T14:16:25,295 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ca89df5d4370400ab1a21973db3ed27e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731766583323 2024-11-16T14:16:25,295 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting f38715c10f984818bce1c74fe60b0942, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731766583323 2024-11-16T14:16:25,295 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 46afb44a3cf745eab4accb8ab318c547, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731766584489 2024-11-16T14:16:25,295 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c44b1f7a40014917999ac872d3607cf0, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731766584489 2024-11-16T14:16:25,301 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:25,301 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#A#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:25,301 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/23416f3859f94991b66468f1ef40f563 is 50, key is test_row_0/A:col10/1731766584839/Put/seqid=0 2024-11-16T14:16:25,301 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/452d27de48c640be8b24c858deb0a8de is 50, key is test_row_0/B:col10/1731766584839/Put/seqid=0 2024-11-16T14:16:25,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742282_1458 (size=12359) 2024-11-16T14:16:25,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742283_1459 (size=12359) 2024-11-16T14:16:25,309 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/452d27de48c640be8b24c858deb0a8de as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/452d27de48c640be8b24c858deb0a8de 2024-11-16T14:16:25,313 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/B of 4824a7508eaae82911857430ab75750c into 452d27de48c640be8b24c858deb0a8de(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:25,313 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:25,313 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/B, priority=13, startTime=1731766585293; duration=0sec 2024-11-16T14:16:25,313 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:25,313 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:B 2024-11-16T14:16:25,313 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:25,320 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:25,320 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/C is initiating minor compaction (all files) 2024-11-16T14:16:25,320 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/C in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:25,320 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/956169663db54b80987c279c7d4b7087, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/10c6f0d941fb40548c1ecef1b841c9f1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/cab720838001492bb32cfb3c176c4b2c] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=35.4 K 2024-11-16T14:16:25,321 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 956169663db54b80987c279c7d4b7087, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731766582110 2024-11-16T14:16:25,321 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 10c6f0d941fb40548c1ecef1b841c9f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731766583323 2024-11-16T14:16:25,321 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting cab720838001492bb32cfb3c176c4b2c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731766584489 2024-11-16T14:16:25,328 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:25,328 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/0d665e3ddb0c4860b11e0c15b2530e92 is 50, key is test_row_0/C:col10/1731766584839/Put/seqid=0 2024-11-16T14:16:25,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742284_1460 (size=12359) 2024-11-16T14:16:25,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T14:16:25,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:25,406 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T14:16:25,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:25,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:25,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:25,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:25,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:25,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:25,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/0f853da2edb3422b974706c4621a4912 is 50, key is test_row_0/A:col10/1731766584872/Put/seqid=0 2024-11-16T14:16:25,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742285_1461 (size=12151) 2024-11-16T14:16:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:25,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:25,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766645530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766645531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766645536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766645537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T14:16:25,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766645645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766645655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766645655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766645656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,711 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/23416f3859f94991b66468f1ef40f563 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/23416f3859f94991b66468f1ef40f563 2024-11-16T14:16:25,714 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/A of 4824a7508eaae82911857430ab75750c into 23416f3859f94991b66468f1ef40f563(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:25,714 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:25,715 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/A, priority=13, startTime=1731766585293; duration=0sec 2024-11-16T14:16:25,715 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:25,715 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:A 2024-11-16T14:16:25,746 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/0d665e3ddb0c4860b11e0c15b2530e92 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0d665e3ddb0c4860b11e0c15b2530e92 2024-11-16T14:16:25,750 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/C of 4824a7508eaae82911857430ab75750c into 0d665e3ddb0c4860b11e0c15b2530e92(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:25,750 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:25,750 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/C, priority=13, startTime=1731766585294; duration=0sec 2024-11-16T14:16:25,750 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:25,750 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:C 2024-11-16T14:16:25,814 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/0f853da2edb3422b974706c4621a4912 2024-11-16T14:16:25,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/c141b8a39e8442f3b027071c2cf648a6 is 50, key is test_row_0/B:col10/1731766584872/Put/seqid=0 2024-11-16T14:16:25,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742286_1462 (size=12151) 2024-11-16T14:16:25,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766645855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766645875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766645876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766645876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T14:16:26,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766646166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:26,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766646185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:26,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766646185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:26,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766646185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:26,222 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/c141b8a39e8442f3b027071c2cf648a6 2024-11-16T14:16:26,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/08480430bb2644d186595a01c94e31f6 is 50, key is test_row_0/C:col10/1731766584872/Put/seqid=0 2024-11-16T14:16:26,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742287_1463 (size=12151) 2024-11-16T14:16:26,632 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/08480430bb2644d186595a01c94e31f6 2024-11-16T14:16:26,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/0f853da2edb3422b974706c4621a4912 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/0f853da2edb3422b974706c4621a4912 2024-11-16T14:16:26,637 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/0f853da2edb3422b974706c4621a4912, entries=150, sequenceid=150, filesize=11.9 K 2024-11-16T14:16:26,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/c141b8a39e8442f3b027071c2cf648a6 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c141b8a39e8442f3b027071c2cf648a6 2024-11-16T14:16:26,641 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c141b8a39e8442f3b027071c2cf648a6, entries=150, sequenceid=150, filesize=11.9 K 2024-11-16T14:16:26,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/08480430bb2644d186595a01c94e31f6 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/08480430bb2644d186595a01c94e31f6 2024-11-16T14:16:26,644 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/08480430bb2644d186595a01c94e31f6, entries=150, sequenceid=150, filesize=11.9 K 2024-11-16T14:16:26,645 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4824a7508eaae82911857430ab75750c in 1239ms, sequenceid=150, compaction requested=false 2024-11-16T14:16:26,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:26,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:26,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-16T14:16:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-16T14:16:26,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-16T14:16:26,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6980 sec 2024-11-16T14:16:26,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.7020 sec 2024-11-16T14:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:26,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T14:16:26,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:26,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:26,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:26,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:26,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:26,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:26,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/b26b0dc52d604665880a7ca1c629d6e9 is 50, key is test_row_0/A:col10/1731766585523/Put/seqid=0 2024-11-16T14:16:26,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742288_1464 (size=14541) 2024-11-16T14:16:26,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766646714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766646715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:26,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766646719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:26,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766646726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:26,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766646836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:26,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:26,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766646836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766646836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766646836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766647045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766647056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766647056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766647056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/b26b0dc52d604665880a7ca1c629d6e9 2024-11-16T14:16:27,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/dda52b6aeb65467b812363318ba22204 is 50, key is test_row_0/B:col10/1731766585523/Put/seqid=0 2024-11-16T14:16:27,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742289_1465 (size=12151) 2024-11-16T14:16:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T14:16:27,096 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:27,096 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-16T14:16:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T14:16:27,097 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:27,098 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:27,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T14:16:27,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T14:16:27,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:27,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:27,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:27,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766647365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766647366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766647376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766647376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T14:16:27,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:27,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:27,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:27,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T14:16:27,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/dda52b6aeb65467b812363318ba22204 2024-11-16T14:16:27,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/c3339c39bcab46b5a717b38bcd544c2b is 50, key is test_row_0/C:col10/1731766585523/Put/seqid=0 2024-11-16T14:16:27,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742290_1466 (size=12151) 2024-11-16T14:16:27,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T14:16:27,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:27,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:27,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:27,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T14:16:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:27,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T14:16:27,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39412 deadline: 1731766647805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,811 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in 4824a7508eaae82911857430ab75750c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T14:16:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:27,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:27,873 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T14:16:27,873 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T14:16:27,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766647875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766647875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766647895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766647895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:27,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/c3339c39bcab46b5a717b38bcd544c2b 2024-11-16T14:16:27,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/b26b0dc52d604665880a7ca1c629d6e9 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/b26b0dc52d604665880a7ca1c629d6e9 2024-11-16T14:16:27,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/b26b0dc52d604665880a7ca1c629d6e9, entries=200, sequenceid=167, filesize=14.2 K 2024-11-16T14:16:27,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/dda52b6aeb65467b812363318ba22204 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/dda52b6aeb65467b812363318ba22204 2024-11-16T14:16:27,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/dda52b6aeb65467b812363318ba22204, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T14:16:27,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/c3339c39bcab46b5a717b38bcd544c2b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c3339c39bcab46b5a717b38bcd544c2b 2024-11-16T14:16:27,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c3339c39bcab46b5a717b38bcd544c2b, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T14:16:27,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4824a7508eaae82911857430ab75750c in 1238ms, sequenceid=167, compaction requested=true 2024-11-16T14:16:27,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:27,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:27,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:27,919 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:27,919 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:27,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:27,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:27,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:27,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:27,920 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:27,920 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:27,920 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/A is initiating minor compaction (all files) 2024-11-16T14:16:27,920 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/B is initiating minor compaction (all files) 2024-11-16T14:16:27,920 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/B in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:27,920 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/A in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:27,920 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/23416f3859f94991b66468f1ef40f563, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/0f853da2edb3422b974706c4621a4912, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/b26b0dc52d604665880a7ca1c629d6e9] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=38.1 K 2024-11-16T14:16:27,920 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/452d27de48c640be8b24c858deb0a8de, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c141b8a39e8442f3b027071c2cf648a6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/dda52b6aeb65467b812363318ba22204] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=35.8 K 2024-11-16T14:16:27,921 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 452d27de48c640be8b24c858deb0a8de, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731766584489 2024-11-16T14:16:27,921 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23416f3859f94991b66468f1ef40f563, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731766584489 2024-11-16T14:16:27,921 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting c141b8a39e8442f3b027071c2cf648a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731766584867 2024-11-16T14:16:27,921 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f853da2edb3422b974706c4621a4912, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731766584867 2024-11-16T14:16:27,921 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting dda52b6aeb65467b812363318ba22204, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731766585523 2024-11-16T14:16:27,921 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b26b0dc52d604665880a7ca1c629d6e9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731766585523 2024-11-16T14:16:27,929 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:27,929 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/013c8981dd484fc6a366c7952fae3c60 is 50, key is test_row_0/A:col10/1731766585523/Put/seqid=0 2024-11-16T14:16:27,929 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:27,930 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/5d36a61d8fc04e7fb0b1cb9a9042e0d0 is 50, key is test_row_0/B:col10/1731766585523/Put/seqid=0 2024-11-16T14:16:27,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742291_1467 (size=12561) 2024-11-16T14:16:27,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742292_1468 (size=12561) 2024-11-16T14:16:28,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T14:16:28,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:28,013 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T14:16:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:28,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/221980f5960d41b2a74d8dbf53083a96 is 50, key is test_row_0/A:col10/1731766586702/Put/seqid=0 2024-11-16T14:16:28,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742293_1469 (size=12151) 2024-11-16T14:16:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T14:16:28,347 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/5d36a61d8fc04e7fb0b1cb9a9042e0d0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/5d36a61d8fc04e7fb0b1cb9a9042e0d0 2024-11-16T14:16:28,347 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/013c8981dd484fc6a366c7952fae3c60 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/013c8981dd484fc6a366c7952fae3c60 2024-11-16T14:16:28,352 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/A of 4824a7508eaae82911857430ab75750c into 013c8981dd484fc6a366c7952fae3c60(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:28,352 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/B of 4824a7508eaae82911857430ab75750c into 5d36a61d8fc04e7fb0b1cb9a9042e0d0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:28,353 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:28,353 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:28,353 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/B, priority=13, startTime=1731766587919; duration=0sec 2024-11-16T14:16:28,353 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/A, priority=13, startTime=1731766587919; duration=0sec 2024-11-16T14:16:28,353 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:28,353 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:28,353 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:B 2024-11-16T14:16:28,353 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:A 2024-11-16T14:16:28,353 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:28,353 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:28,354 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/C is initiating minor compaction (all files) 2024-11-16T14:16:28,354 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/C in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:28,354 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0d665e3ddb0c4860b11e0c15b2530e92, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/08480430bb2644d186595a01c94e31f6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c3339c39bcab46b5a717b38bcd544c2b] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=35.8 K 2024-11-16T14:16:28,354 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d665e3ddb0c4860b11e0c15b2530e92, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731766584489 2024-11-16T14:16:28,354 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 08480430bb2644d186595a01c94e31f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731766584867 2024-11-16T14:16:28,354 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting c3339c39bcab46b5a717b38bcd544c2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731766585523 2024-11-16T14:16:28,364 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#C#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:28,365 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/5bd449c0d9044282b76dffbf12df5ee2 is 50, key is test_row_0/C:col10/1731766585523/Put/seqid=0 2024-11-16T14:16:28,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742294_1470 (size=12561) 2024-11-16T14:16:28,381 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/5bd449c0d9044282b76dffbf12df5ee2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5bd449c0d9044282b76dffbf12df5ee2 2024-11-16T14:16:28,384 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/C of 4824a7508eaae82911857430ab75750c into 5bd449c0d9044282b76dffbf12df5ee2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:28,384 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:28,384 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/C, priority=13, startTime=1731766587919; duration=0sec 2024-11-16T14:16:28,384 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:28,384 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:C 2024-11-16T14:16:28,421 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/221980f5960d41b2a74d8dbf53083a96 2024-11-16T14:16:28,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/9088ef7c72f145429a1a09cf1969a0c6 is 50, key is test_row_0/B:col10/1731766586702/Put/seqid=0 2024-11-16T14:16:28,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742295_1471 (size=12151) 2024-11-16T14:16:28,837 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/9088ef7c72f145429a1a09cf1969a0c6 2024-11-16T14:16:28,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/e6a1b547001a4a88a47af958bb3481ad is 50, key is test_row_0/C:col10/1731766586702/Put/seqid=0 2024-11-16T14:16:28,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742296_1472 (size=12151) 2024-11-16T14:16:28,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:28,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766648935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:28,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766648938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:28,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766648938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:28,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766648940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:28,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:28,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:28,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766649045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766649056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766649056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766649057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T14:16:29,248 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/e6a1b547001a4a88a47af958bb3481ad 2024-11-16T14:16:29,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/221980f5960d41b2a74d8dbf53083a96 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/221980f5960d41b2a74d8dbf53083a96 2024-11-16T14:16:29,254 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/221980f5960d41b2a74d8dbf53083a96, entries=150, sequenceid=184, filesize=11.9 K 2024-11-16T14:16:29,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/9088ef7c72f145429a1a09cf1969a0c6 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9088ef7c72f145429a1a09cf1969a0c6 2024-11-16T14:16:29,268 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9088ef7c72f145429a1a09cf1969a0c6, entries=150, sequenceid=184, filesize=11.9 K 2024-11-16T14:16:29,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/e6a1b547001a4a88a47af958bb3481ad as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/e6a1b547001a4a88a47af958bb3481ad 2024-11-16T14:16:29,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766649266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,273 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/e6a1b547001a4a88a47af958bb3481ad, entries=150, sequenceid=184, filesize=11.9 K 2024-11-16T14:16:29,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766649266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,274 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4824a7508eaae82911857430ab75750c in 1260ms, sequenceid=184, compaction requested=false 2024-11-16T14:16:29,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:29,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:29,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-16T14:16:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-16T14:16:29,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-16T14:16:29,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1770 sec 2024-11-16T14:16:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:29,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T14:16:29,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:29,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:29,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:29,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:29,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:29,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:29,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.1830 sec 2024-11-16T14:16:29,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/847653016ad84ee29033de50a2891af1 is 50, key is test_row_0/A:col10/1731766588933/Put/seqid=0 2024-11-16T14:16:29,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742297_1473 (size=14541) 2024-11-16T14:16:29,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766649342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766649354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766649465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766649476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766649585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766649585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766649675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766649695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:29,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/847653016ad84ee29033de50a2891af1 2024-11-16T14:16:29,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/b26b24dd9d6a401f9c4d1666449d6635 is 50, key is test_row_0/B:col10/1731766588933/Put/seqid=0 2024-11-16T14:16:29,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742298_1474 (size=12151) 2024-11-16T14:16:29,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/b26b24dd9d6a401f9c4d1666449d6635 2024-11-16T14:16:29,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/b4c46ed48880499da696c457bcbfc09e is 50, key is test_row_0/C:col10/1731766588933/Put/seqid=0 2024-11-16T14:16:29,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742299_1475 (size=12151) 2024-11-16T14:16:30,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766649995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:30,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766650016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:30,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766650109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:30,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766650109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:30,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/b4c46ed48880499da696c457bcbfc09e 2024-11-16T14:16:30,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/847653016ad84ee29033de50a2891af1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/847653016ad84ee29033de50a2891af1 2024-11-16T14:16:30,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/847653016ad84ee29033de50a2891af1, entries=200, sequenceid=203, filesize=14.2 K 2024-11-16T14:16:30,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/b26b24dd9d6a401f9c4d1666449d6635 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b26b24dd9d6a401f9c4d1666449d6635 2024-11-16T14:16:30,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b26b24dd9d6a401f9c4d1666449d6635, entries=150, sequenceid=203, filesize=11.9 K 2024-11-16T14:16:30,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/b4c46ed48880499da696c457bcbfc09e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/b4c46ed48880499da696c457bcbfc09e 2024-11-16T14:16:30,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/b4c46ed48880499da696c457bcbfc09e, entries=150, sequenceid=203, filesize=11.9 K 2024-11-16T14:16:30,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4824a7508eaae82911857430ab75750c in 990ms, sequenceid=203, compaction requested=true 2024-11-16T14:16:30,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:30,271 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:30,272 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:30,272 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/A is initiating minor compaction (all files) 2024-11-16T14:16:30,272 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/A in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:30,272 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/013c8981dd484fc6a366c7952fae3c60, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/221980f5960d41b2a74d8dbf53083a96, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/847653016ad84ee29033de50a2891af1] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=38.3 K 2024-11-16T14:16:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:30,273 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:30,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:30,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:30,275 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 013c8981dd484fc6a366c7952fae3c60, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731766585523 2024-11-16T14:16:30,276 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221980f5960d41b2a74d8dbf53083a96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731766586702 2024-11-16T14:16:30,278 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 847653016ad84ee29033de50a2891af1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766588917 2024-11-16T14:16:30,278 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:30,278 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/B is initiating minor compaction (all files) 2024-11-16T14:16:30,279 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/B in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:30,279 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/5d36a61d8fc04e7fb0b1cb9a9042e0d0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9088ef7c72f145429a1a09cf1969a0c6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b26b24dd9d6a401f9c4d1666449d6635] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=36.0 K 2024-11-16T14:16:30,279 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d36a61d8fc04e7fb0b1cb9a9042e0d0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731766585523 2024-11-16T14:16:30,280 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9088ef7c72f145429a1a09cf1969a0c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731766586702 2024-11-16T14:16:30,280 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting b26b24dd9d6a401f9c4d1666449d6635, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766588917 2024-11-16T14:16:30,299 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:30,299 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/d93c520bfbf6473f85f667c053aff23a is 50, key is test_row_0/A:col10/1731766588933/Put/seqid=0 2024-11-16T14:16:30,310 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:30,310 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/622a0594a0894943a044215970dd76a9 is 50, key is test_row_0/B:col10/1731766588933/Put/seqid=0 2024-11-16T14:16:30,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742300_1476 (size=12663) 2024-11-16T14:16:30,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742301_1477 (size=12663) 2024-11-16T14:16:30,349 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/622a0594a0894943a044215970dd76a9 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/622a0594a0894943a044215970dd76a9 2024-11-16T14:16:30,363 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/B of 4824a7508eaae82911857430ab75750c into 622a0594a0894943a044215970dd76a9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:30,363 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:30,363 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/B, priority=13, startTime=1731766590273; duration=0sec 2024-11-16T14:16:30,364 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:30,364 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:B 2024-11-16T14:16:30,364 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:30,365 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:30,365 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/C is initiating minor compaction (all files) 2024-11-16T14:16:30,365 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/C in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:30,365 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5bd449c0d9044282b76dffbf12df5ee2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/e6a1b547001a4a88a47af958bb3481ad, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/b4c46ed48880499da696c457bcbfc09e] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=36.0 K 2024-11-16T14:16:30,365 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd449c0d9044282b76dffbf12df5ee2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731766585523 2024-11-16T14:16:30,365 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting e6a1b547001a4a88a47af958bb3481ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731766586702 2024-11-16T14:16:30,366 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting b4c46ed48880499da696c457bcbfc09e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766588917 2024-11-16T14:16:30,381 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:30,382 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/b4809c8ac5a14d4281c6dc095b3ed77d is 50, key is test_row_0/C:col10/1731766588933/Put/seqid=0 2024-11-16T14:16:30,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742302_1478 (size=12663) 2024-11-16T14:16:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:30,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T14:16:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:30,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/ba5b057cfaf04eb18d0522a19a2b08cd is 50, key is test_row_0/A:col10/1731766589340/Put/seqid=0 2024-11-16T14:16:30,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742303_1479 (size=14541) 2024-11-16T14:16:30,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/ba5b057cfaf04eb18d0522a19a2b08cd 2024-11-16T14:16:30,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/f1294a761fb345e4aae656028215b40c is 50, key is test_row_0/B:col10/1731766589340/Put/seqid=0 2024-11-16T14:16:30,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742304_1480 (size=12151) 2024-11-16T14:16:30,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/f1294a761fb345e4aae656028215b40c 2024-11-16T14:16:30,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/6cade305c68b4ace93c863ee3e789ef9 is 50, key is test_row_0/C:col10/1731766589340/Put/seqid=0 2024-11-16T14:16:30,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742305_1481 (size=12151) 2024-11-16T14:16:30,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/6cade305c68b4ace93c863ee3e789ef9 2024-11-16T14:16:30,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/ba5b057cfaf04eb18d0522a19a2b08cd as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/ba5b057cfaf04eb18d0522a19a2b08cd 2024-11-16T14:16:30,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/ba5b057cfaf04eb18d0522a19a2b08cd, entries=200, sequenceid=221, filesize=14.2 K 2024-11-16T14:16:30,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/f1294a761fb345e4aae656028215b40c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/f1294a761fb345e4aae656028215b40c 2024-11-16T14:16:30,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/f1294a761fb345e4aae656028215b40c, entries=150, sequenceid=221, filesize=11.9 K 2024-11-16T14:16:30,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/6cade305c68b4ace93c863ee3e789ef9 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/6cade305c68b4ace93c863ee3e789ef9 2024-11-16T14:16:30,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/6cade305c68b4ace93c863ee3e789ef9, entries=150, sequenceid=221, filesize=11.9 K 2024-11-16T14:16:30,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=26.84 KB/27480 for 4824a7508eaae82911857430ab75750c in 88ms, sequenceid=221, compaction requested=false 2024-11-16T14:16:30,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:30,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:16:30,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:30,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:30,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:30,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:30,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:30,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:30,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/fe051e4808d84752a77818d4bba068b8 is 50, key is test_row_0/A:col10/1731766590628/Put/seqid=0 2024-11-16T14:16:30,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742306_1482 (size=14541) 2024-11-16T14:16:30,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766590691, max timestamp=1731766590712Num uniques 859; 2024-11-16T14:16:30,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766590691, max timestamp=1731766590712Num uniques 859; 2024-11-16T14:16:30,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766590691, max timestamp=1731766590712Num uniques 859; 2024-11-16T14:16:30,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:16:30,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:16:30,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:16:30,735 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/d93c520bfbf6473f85f667c053aff23a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/d93c520bfbf6473f85f667c053aff23a 2024-11-16T14:16:30,743 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/A of 4824a7508eaae82911857430ab75750c into d93c520bfbf6473f85f667c053aff23a(size=12.4 K), total size for store is 26.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:30,743 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:30,743 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/A, priority=13, startTime=1731766590271; duration=0sec 2024-11-16T14:16:30,743 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:30,743 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:A 2024-11-16T14:16:30,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766650816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:30,838 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/b4809c8ac5a14d4281c6dc095b3ed77d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/b4809c8ac5a14d4281c6dc095b3ed77d 2024-11-16T14:16:30,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766650827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:30,845 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/C of 4824a7508eaae82911857430ab75750c into b4809c8ac5a14d4281c6dc095b3ed77d(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:30,845 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:30,845 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/C, priority=13, startTime=1731766590273; duration=0sec 2024-11-16T14:16:30,845 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:30,845 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:C 2024-11-16T14:16:30,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766650935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:30,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766650947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:31,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/fe051e4808d84752a77818d4bba068b8 2024-11-16T14:16:31,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/d4f5efe847d94c58bc7a3a112b87d777 is 50, key is test_row_0/B:col10/1731766590628/Put/seqid=0 2024-11-16T14:16:31,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742307_1483 (size=12151) 2024-11-16T14:16:31,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/d4f5efe847d94c58bc7a3a112b87d777 2024-11-16T14:16:31,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/1782bf2079ed41908409235e0900c922 is 50, key is test_row_0/C:col10/1731766590628/Put/seqid=0 2024-11-16T14:16:31,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742308_1484 (size=12151) 2024-11-16T14:16:31,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766651126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:31,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766651126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:31,126 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in 4824a7508eaae82911857430ab75750c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:31,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:31,127 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in 4824a7508eaae82911857430ab75750c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:31,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:31,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:31,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:31,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:31,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:31,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766651156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:31,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:31,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:31,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766651156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:31,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:31,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:31,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:31,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T14:16:31,266 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:31,266 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-16T14:16:31,268 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T14:16:31,269 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:31,269 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T14:16:31,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:16:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:31,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:31,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766651466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:31,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766651466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:31,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:31,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:31,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:31,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/1782bf2079ed41908409235e0900c922 2024-11-16T14:16:31,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/fe051e4808d84752a77818d4bba068b8 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/fe051e4808d84752a77818d4bba068b8 2024-11-16T14:16:31,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/fe051e4808d84752a77818d4bba068b8, entries=200, sequenceid=231, filesize=14.2 K 2024-11-16T14:16:31,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/d4f5efe847d94c58bc7a3a112b87d777 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/d4f5efe847d94c58bc7a3a112b87d777 2024-11-16T14:16:31,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/d4f5efe847d94c58bc7a3a112b87d777, entries=150, sequenceid=231, filesize=11.9 K 2024-11-16T14:16:31,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/1782bf2079ed41908409235e0900c922 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/1782bf2079ed41908409235e0900c922 2024-11-16T14:16:31,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/1782bf2079ed41908409235e0900c922, entries=150, sequenceid=231, filesize=11.9 K 2024-11-16T14:16:31,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4824a7508eaae82911857430ab75750c in 869ms, sequenceid=231, compaction requested=true 2024-11-16T14:16:31,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:31,498 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:31,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:31,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:31,498 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:31,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:31,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:31,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:31,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:31,520 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:31,520 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/A is initiating minor compaction (all files) 2024-11-16T14:16:31,520 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/A in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:31,520 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/d93c520bfbf6473f85f667c053aff23a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/ba5b057cfaf04eb18d0522a19a2b08cd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/fe051e4808d84752a77818d4bba068b8] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=40.8 K 2024-11-16T14:16:31,520 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:31,521 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/B is initiating minor compaction (all files) 2024-11-16T14:16:31,521 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/B in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:31,521 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/622a0594a0894943a044215970dd76a9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/f1294a761fb345e4aae656028215b40c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/d4f5efe847d94c58bc7a3a112b87d777] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=36.1 K 2024-11-16T14:16:31,521 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d93c520bfbf6473f85f667c053aff23a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766588917 2024-11-16T14:16:31,521 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 622a0594a0894943a044215970dd76a9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766588917 2024-11-16T14:16:31,521 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba5b057cfaf04eb18d0522a19a2b08cd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731766589299 2024-11-16T14:16:31,522 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f1294a761fb345e4aae656028215b40c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731766589299 2024-11-16T14:16:31,524 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting d4f5efe847d94c58bc7a3a112b87d777, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731766590594 2024-11-16T14:16:31,524 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe051e4808d84752a77818d4bba068b8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731766590590 2024-11-16T14:16:31,543 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#B#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:31,544 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/3af28a2982364482982c367f03ec4057 is 50, key is test_row_0/B:col10/1731766590628/Put/seqid=0 2024-11-16T14:16:31,551 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#A#compaction#415 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:31,551 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/3311f601f44743f38b4b2fbc805c7167 is 50, key is test_row_0/A:col10/1731766590628/Put/seqid=0 2024-11-16T14:16:31,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742309_1485 (size=12765) 2024-11-16T14:16:31,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742310_1486 (size=12765) 2024-11-16T14:16:31,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:16:31,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:31,575 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T14:16:31,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:31,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:31,575 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/3311f601f44743f38b4b2fbc805c7167 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3311f601f44743f38b4b2fbc805c7167 2024-11-16T14:16:31,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:31,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:31,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:31,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T14:16:31,588 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/A of 4824a7508eaae82911857430ab75750c into 3311f601f44743f38b4b2fbc805c7167(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:31,588 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:31,588 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/A, priority=13, startTime=1731766591498; duration=0sec 2024-11-16T14:16:31,588 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:31,588 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:A 2024-11-16T14:16:31,589 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:31,590 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:31,590 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/C is initiating minor compaction (all files) 2024-11-16T14:16:31,590 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/C in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:31,590 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/b4809c8ac5a14d4281c6dc095b3ed77d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/6cade305c68b4ace93c863ee3e789ef9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/1782bf2079ed41908409235e0900c922] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=36.1 K 2024-11-16T14:16:31,590 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4809c8ac5a14d4281c6dc095b3ed77d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731766588917 2024-11-16T14:16:31,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/e1ee48ffb0ff4fb09d5724aceb94d43f is 50, key is test_row_0/A:col10/1731766590794/Put/seqid=0 2024-11-16T14:16:31,590 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cade305c68b4ace93c863ee3e789ef9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731766589299 2024-11-16T14:16:31,591 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1782bf2079ed41908409235e0900c922, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731766590594 2024-11-16T14:16:31,594 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/3af28a2982364482982c367f03ec4057 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/3af28a2982364482982c367f03ec4057 2024-11-16T14:16:31,602 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/B of 4824a7508eaae82911857430ab75750c into 3af28a2982364482982c367f03ec4057(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:31,602 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:31,602 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/B, priority=13, startTime=1731766591498; duration=0sec 2024-11-16T14:16:31,603 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:31,603 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:B 2024-11-16T14:16:31,606 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:31,607 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/4e6689c444d8494e95e632f8c9eec221 is 50, key is test_row_0/C:col10/1731766590628/Put/seqid=0 2024-11-16T14:16:31,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742311_1487 (size=12301) 2024-11-16T14:16:31,622 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/e1ee48ffb0ff4fb09d5724aceb94d43f 2024-11-16T14:16:31,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742312_1488 (size=12765) 2024-11-16T14:16:31,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/16a288940c08465799804560f802b676 is 50, key is test_row_0/B:col10/1731766590794/Put/seqid=0 2024-11-16T14:16:31,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742313_1489 (size=12301) 2024-11-16T14:16:31,689 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/16a288940c08465799804560f802b676 2024-11-16T14:16:31,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/c60410941ae44ff8bb9eef17f6642e1d is 50, key is test_row_0/C:col10/1731766590794/Put/seqid=0 2024-11-16T14:16:31,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742314_1490 (size=12301) 2024-11-16T14:16:31,714 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/c60410941ae44ff8bb9eef17f6642e1d 2024-11-16T14:16:31,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/e1ee48ffb0ff4fb09d5724aceb94d43f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/e1ee48ffb0ff4fb09d5724aceb94d43f 2024-11-16T14:16:31,722 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/e1ee48ffb0ff4fb09d5724aceb94d43f, entries=150, sequenceid=270, filesize=12.0 K 2024-11-16T14:16:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/16a288940c08465799804560f802b676 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/16a288940c08465799804560f802b676 2024-11-16T14:16:31,725 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/16a288940c08465799804560f802b676, entries=150, sequenceid=270, filesize=12.0 K 2024-11-16T14:16:31,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/c60410941ae44ff8bb9eef17f6642e1d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c60410941ae44ff8bb9eef17f6642e1d 2024-11-16T14:16:31,736 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c60410941ae44ff8bb9eef17f6642e1d, entries=150, sequenceid=270, filesize=12.0 K 2024-11-16T14:16:31,738 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for 4824a7508eaae82911857430ab75750c in 162ms, sequenceid=270, compaction requested=false 2024-11-16T14:16:31,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:31,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:31,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-16T14:16:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-16T14:16:31,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-16T14:16:31,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 470 msec 2024-11-16T14:16:31,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 476 msec 2024-11-16T14:16:31,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T14:16:31,896 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:31,896 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:31,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-16T14:16:31,898 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T14:16:31,898 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:31,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:31,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:16:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:32,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/3e8cbc53f7a249069ca12c8b5575698c is 50, key is test_row_0/A:col10/1731766591989/Put/seqid=0 2024-11-16T14:16:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T14:16:32,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742315_1491 (size=17177) 2024-11-16T14:16:32,049 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/4e6689c444d8494e95e632f8c9eec221 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4e6689c444d8494e95e632f8c9eec221 2024-11-16T14:16:32,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T14:16:32,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:32,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:32,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:32,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:32,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:32,056 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/C of 4824a7508eaae82911857430ab75750c into 4e6689c444d8494e95e632f8c9eec221(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:32,056 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:32,056 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/C, priority=13, startTime=1731766591498; duration=0sec 2024-11-16T14:16:32,056 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:32,056 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:C 2024-11-16T14:16:32,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731766592092, max timestamp=1731766592107Num uniques 859; 2024-11-16T14:16:32,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:16:32,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731766592092, max timestamp=1731766592107Num uniques 859; 2024-11-16T14:16:32,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:16:32,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731766592092, max timestamp=1731766592107Num uniques 859; 2024-11-16T14:16:32,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:16:32,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766652177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:32,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766652180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:32,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T14:16:32,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:32,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:32,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:32,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:32,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T14:16:32,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766652295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:32,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766652306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:32,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T14:16:32,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:32,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:32,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:32,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:32,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:32,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:32,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/3e8cbc53f7a249069ca12c8b5575698c 2024-11-16T14:16:32,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/caae545d9f244ecd86c2ccdf843f3feb is 50, key is test_row_0/B:col10/1731766591989/Put/seqid=0 2024-11-16T14:16:32,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742316_1492 (size=9857) 2024-11-16T14:16:32,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/caae545d9f244ecd86c2ccdf843f3feb 2024-11-16T14:16:32,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/5da89011ffc8438d953fcd246313678a is 50, key is test_row_0/C:col10/1731766591989/Put/seqid=0 2024-11-16T14:16:32,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742317_1493 (size=9857) 2024-11-16T14:16:32,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/5da89011ffc8438d953fcd246313678a 2024-11-16T14:16:32,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/3e8cbc53f7a249069ca12c8b5575698c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3e8cbc53f7a249069ca12c8b5575698c 2024-11-16T14:16:32,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3e8cbc53f7a249069ca12c8b5575698c, entries=250, sequenceid=282, filesize=16.8 K 2024-11-16T14:16:32,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/caae545d9f244ecd86c2ccdf843f3feb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/caae545d9f244ecd86c2ccdf843f3feb 2024-11-16T14:16:32,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/caae545d9f244ecd86c2ccdf843f3feb, entries=100, sequenceid=282, filesize=9.6 K 2024-11-16T14:16:32,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/5da89011ffc8438d953fcd246313678a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5da89011ffc8438d953fcd246313678a 2024-11-16T14:16:32,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5da89011ffc8438d953fcd246313678a, entries=100, sequenceid=282, filesize=9.6 K 2024-11-16T14:16:32,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4824a7508eaae82911857430ab75750c in 486ms, sequenceid=282, compaction requested=true 2024-11-16T14:16:32,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:32,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:32,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:32,483 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:32,483 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:32,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:32,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:32,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:32,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:32,487 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:32,487 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/A is initiating minor compaction (all files) 2024-11-16T14:16:32,487 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/A in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:32,488 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3311f601f44743f38b4b2fbc805c7167, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/e1ee48ffb0ff4fb09d5724aceb94d43f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3e8cbc53f7a249069ca12c8b5575698c] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=41.3 K 2024-11-16T14:16:32,488 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:32,488 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/B is initiating minor compaction (all files) 2024-11-16T14:16:32,488 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/B in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:32,488 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/3af28a2982364482982c367f03ec4057, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/16a288940c08465799804560f802b676, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/caae545d9f244ecd86c2ccdf843f3feb] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=34.1 K 2024-11-16T14:16:32,489 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af28a2982364482982c367f03ec4057, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731766590594 2024-11-16T14:16:32,491 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3311f601f44743f38b4b2fbc805c7167, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731766590594 2024-11-16T14:16:32,492 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 16a288940c08465799804560f802b676, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731766590794 2024-11-16T14:16:32,492 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1ee48ffb0ff4fb09d5724aceb94d43f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731766590794 2024-11-16T14:16:32,492 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e8cbc53f7a249069ca12c8b5575698c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731766591989 2024-11-16T14:16:32,492 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting caae545d9f244ecd86c2ccdf843f3feb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731766591989 2024-11-16T14:16:32,510 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#B#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:32,510 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/060ada14e1484da3881450a5ba98f487 is 50, key is test_row_0/B:col10/1731766591989/Put/seqid=0 2024-11-16T14:16:32,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T14:16:32,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:32,512 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-16T14:16:32,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:32,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:32,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:32,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:32,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:32,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:32,518 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#A#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:32,519 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/dc572925d99242b18d3dfc5d7b94e39f is 50, key is test_row_0/A:col10/1731766591989/Put/seqid=0 2024-11-16T14:16:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T14:16:32,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/d3a599f3c6a64b8ca64c8f44cc437f05 is 50, key is test_row_0/A:col10/1731766592175/Put/seqid=0 2024-11-16T14:16:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:32,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:32,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742319_1495 (size=13017) 2024-11-16T14:16:32,550 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/dc572925d99242b18d3dfc5d7b94e39f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/dc572925d99242b18d3dfc5d7b94e39f 2024-11-16T14:16:32,560 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/A of 4824a7508eaae82911857430ab75750c into dc572925d99242b18d3dfc5d7b94e39f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:32,560 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:32,560 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/A, priority=13, startTime=1731766592482; duration=0sec 2024-11-16T14:16:32,561 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:32,561 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:A 2024-11-16T14:16:32,561 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:32,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742318_1494 (size=13017) 2024-11-16T14:16:32,561 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:32,562 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/C is initiating minor compaction (all files) 2024-11-16T14:16:32,562 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/C in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:32,562 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4e6689c444d8494e95e632f8c9eec221, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c60410941ae44ff8bb9eef17f6642e1d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5da89011ffc8438d953fcd246313678a] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=34.1 K 2024-11-16T14:16:32,563 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e6689c444d8494e95e632f8c9eec221, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731766590594 2024-11-16T14:16:32,563 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting c60410941ae44ff8bb9eef17f6642e1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731766590794 2024-11-16T14:16:32,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742320_1496 (size=12301) 2024-11-16T14:16:32,565 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5da89011ffc8438d953fcd246313678a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731766591989 2024-11-16T14:16:32,583 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#C#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:32,583 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/ce77b5c6b9b64e7b8e0711d95d007187 is 50, key is test_row_0/C:col10/1731766591989/Put/seqid=0 2024-11-16T14:16:32,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766652572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:32,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742321_1497 (size=13017) 2024-11-16T14:16:32,606 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/ce77b5c6b9b64e7b8e0711d95d007187 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/ce77b5c6b9b64e7b8e0711d95d007187 2024-11-16T14:16:32,609 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/C of 4824a7508eaae82911857430ab75750c into ce77b5c6b9b64e7b8e0711d95d007187(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:32,609 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:32,609 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/C, priority=13, startTime=1731766592483; duration=0sec 2024-11-16T14:16:32,609 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:32,609 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:C 2024-11-16T14:16:32,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766652590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:32,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:32,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766652697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:32,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:32,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766652716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:32,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766652917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:32,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766652937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:32,964 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/060ada14e1484da3881450a5ba98f487 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/060ada14e1484da3881450a5ba98f487 2024-11-16T14:16:32,966 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/d3a599f3c6a64b8ca64c8f44cc437f05 2024-11-16T14:16:32,969 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/B of 4824a7508eaae82911857430ab75750c into 060ada14e1484da3881450a5ba98f487(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:32,969 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:32,969 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/B, priority=13, startTime=1731766592483; duration=0sec 2024-11-16T14:16:32,969 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:32,969 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:B 2024-11-16T14:16:32,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/b12de2125fd1485cbd26b3db1c79bdd4 is 50, key is test_row_0/B:col10/1731766592175/Put/seqid=0 2024-11-16T14:16:32,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742322_1498 (size=12301) 2024-11-16T14:16:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T14:16:33,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766653136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:33,146 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in 4824a7508eaae82911857430ab75750c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:33,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766653145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:33,148 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in 4824a7508eaae82911857430ab75750c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:33,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:33,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766653226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:33,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:33,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766653255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:33,382 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/b12de2125fd1485cbd26b3db1c79bdd4 2024-11-16T14:16:33,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/77411ebf37714f9da6b709549b14225d is 50, key is test_row_0/C:col10/1731766592175/Put/seqid=0 2024-11-16T14:16:33,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742323_1499 (size=12301) 2024-11-16T14:16:33,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766653746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:33,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:33,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:33,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:33,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766653765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:33,794 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/77411ebf37714f9da6b709549b14225d 2024-11-16T14:16:33,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/d3a599f3c6a64b8ca64c8f44cc437f05 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/d3a599f3c6a64b8ca64c8f44cc437f05 2024-11-16T14:16:33,803 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/d3a599f3c6a64b8ca64c8f44cc437f05, entries=150, sequenceid=319, filesize=12.0 K 2024-11-16T14:16:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/b12de2125fd1485cbd26b3db1c79bdd4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b12de2125fd1485cbd26b3db1c79bdd4 2024-11-16T14:16:33,807 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b12de2125fd1485cbd26b3db1c79bdd4, entries=150, sequenceid=319, filesize=12.0 K 2024-11-16T14:16:33,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/77411ebf37714f9da6b709549b14225d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/77411ebf37714f9da6b709549b14225d 2024-11-16T14:16:33,811 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/77411ebf37714f9da6b709549b14225d, entries=150, sequenceid=319, filesize=12.0 K 2024-11-16T14:16:33,812 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 4824a7508eaae82911857430ab75750c in 1300ms, sequenceid=319, compaction requested=false 2024-11-16T14:16:33,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:33,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:33,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-16T14:16:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-16T14:16:33,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-16T14:16:33,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9170 sec 2024-11-16T14:16:33,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9230 sec 2024-11-16T14:16:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T14:16:34,046 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:34,047 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-16T14:16:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T14:16:34,048 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:34,049 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:34,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:34,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T14:16:34,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T14:16:34,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:34,201 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:16:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/8b954ccd6fe4459b92778ee017f95904 is 50, key is test_row_0/A:col10/1731766592585/Put/seqid=0 2024-11-16T14:16:34,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742324_1500 (size=9857) 2024-11-16T14:16:34,213 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/8b954ccd6fe4459b92778ee017f95904 2024-11-16T14:16:34,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/b8a811abd6df4986a3096202800ed80b is 50, key is test_row_0/B:col10/1731766592585/Put/seqid=0 2024-11-16T14:16:34,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742325_1501 (size=9857) 2024-11-16T14:16:34,237 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/b8a811abd6df4986a3096202800ed80b 2024-11-16T14:16:34,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/7c1ee7f2a14048a1b6d47bb1d912dc1d is 50, key is test_row_0/C:col10/1731766592585/Put/seqid=0 2024-11-16T14:16:34,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742326_1502 (size=9857) 2024-11-16T14:16:34,276 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/7c1ee7f2a14048a1b6d47bb1d912dc1d 2024-11-16T14:16:34,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/8b954ccd6fe4459b92778ee017f95904 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/8b954ccd6fe4459b92778ee017f95904 2024-11-16T14:16:34,285 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/8b954ccd6fe4459b92778ee017f95904, entries=100, sequenceid=332, filesize=9.6 K 2024-11-16T14:16:34,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/b8a811abd6df4986a3096202800ed80b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b8a811abd6df4986a3096202800ed80b 2024-11-16T14:16:34,291 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b8a811abd6df4986a3096202800ed80b, entries=100, sequenceid=332, filesize=9.6 K 2024-11-16T14:16:34,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/7c1ee7f2a14048a1b6d47bb1d912dc1d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/7c1ee7f2a14048a1b6d47bb1d912dc1d 2024-11-16T14:16:34,296 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/7c1ee7f2a14048a1b6d47bb1d912dc1d, entries=100, sequenceid=332, filesize=9.6 K 2024-11-16T14:16:34,296 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 4824a7508eaae82911857430ab75750c in 95ms, sequenceid=332, compaction requested=true 2024-11-16T14:16:34,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:34,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:34,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-16T14:16:34,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-16T14:16:34,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-16T14:16:34,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 251 msec 2024-11-16T14:16:34,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 255 msec 2024-11-16T14:16:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T14:16:34,366 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:34,366 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:34,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-16T14:16:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T14:16:34,369 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:34,370 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:34,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:34,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T14:16:34,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T14:16:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-16T14:16:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-16T14:16:34,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-16T14:16:34,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-16T14:16:34,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 161 msec 2024-11-16T14:16:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T14:16:34,685 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:34,686 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-16T14:16:34,688 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:34,688 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:34,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T14:16:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T14:16:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:34,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:16:34,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:34,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:34,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:34,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:34,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:34,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:34,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/67ef39a320424759935194995e2b3c64 is 50, key is test_row_0/A:col10/1731766594788/Put/seqid=0 2024-11-16T14:16:34,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742327_1503 (size=14741) 2024-11-16T14:16:34,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T14:16:34,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:34,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:34,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:34,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:34,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:34,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766594847, max timestamp=1731766594879Num uniques 859; 2024-11-16T14:16:34,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:16:34,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766594847, max timestamp=1731766594879Num uniques 859; 2024-11-16T14:16:34,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:16:34,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766594847, max timestamp=1731766594879Num uniques 859; 2024-11-16T14:16:34,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:16:34,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766654983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:34,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:34,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:34,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766654984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:34,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T14:16:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:34,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T14:16:35,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766655096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:35,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766655105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:35,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:35,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:35,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:35,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T14:16:35,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:35,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:35,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:35,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/67ef39a320424759935194995e2b3c64 2024-11-16T14:16:35,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/144f9b34516446779e47c2e2f62c32c1 is 50, key is test_row_0/B:col10/1731766594788/Put/seqid=0 2024-11-16T14:16:35,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742328_1504 (size=12301) 2024-11-16T14:16:35,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T14:16:35,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:35,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:35,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:35,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T14:16:35,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766655316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:35,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766655325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:35,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T14:16:35,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:35,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:35,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:35,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T14:16:35,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:35,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:35,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:35,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:35,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/144f9b34516446779e47c2e2f62c32c1 2024-11-16T14:16:35,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/73f1e464146246d7b7b93d7f020b04df is 50, key is test_row_0/C:col10/1731766594788/Put/seqid=0 2024-11-16T14:16:35,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742329_1505 (size=12301) 2024-11-16T14:16:35,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/73f1e464146246d7b7b93d7f020b04df 2024-11-16T14:16:35,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766655626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:35,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:35,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:35,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:35,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/67ef39a320424759935194995e2b3c64 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/67ef39a320424759935194995e2b3c64 2024-11-16T14:16:35,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766655646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:35,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:35,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:35,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:35,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/67ef39a320424759935194995e2b3c64, entries=200, sequenceid=343, filesize=14.4 K 2024-11-16T14:16:35,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/144f9b34516446779e47c2e2f62c32c1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/144f9b34516446779e47c2e2f62c32c1 2024-11-16T14:16:35,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/144f9b34516446779e47c2e2f62c32c1, entries=150, sequenceid=343, filesize=12.0 K 2024-11-16T14:16:35,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/73f1e464146246d7b7b93d7f020b04df as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/73f1e464146246d7b7b93d7f020b04df 2024-11-16T14:16:35,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/73f1e464146246d7b7b93d7f020b04df, entries=150, sequenceid=343, filesize=12.0 K 2024-11-16T14:16:35,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4824a7508eaae82911857430ab75750c in 873ms, sequenceid=343, compaction requested=true 2024-11-16T14:16:35,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:35,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:35,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:35,674 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:16:35,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:35,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:35,674 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:16:35,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:35,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:35,675 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49916 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:16:35,675 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:16:35,675 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/A is initiating minor compaction (all files) 2024-11-16T14:16:35,675 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/B is initiating minor compaction (all files) 2024-11-16T14:16:35,676 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/A in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:35,676 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/B in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:35,676 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/dc572925d99242b18d3dfc5d7b94e39f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/d3a599f3c6a64b8ca64c8f44cc437f05, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/8b954ccd6fe4459b92778ee017f95904, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/67ef39a320424759935194995e2b3c64] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=48.7 K 2024-11-16T14:16:35,676 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/060ada14e1484da3881450a5ba98f487, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b12de2125fd1485cbd26b3db1c79bdd4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b8a811abd6df4986a3096202800ed80b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/144f9b34516446779e47c2e2f62c32c1] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=46.4 K 2024-11-16T14:16:35,676 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 060ada14e1484da3881450a5ba98f487, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731766590807 2024-11-16T14:16:35,676 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc572925d99242b18d3dfc5d7b94e39f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731766590807 2024-11-16T14:16:35,676 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting b12de2125fd1485cbd26b3db1c79bdd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731766592158 2024-11-16T14:16:35,677 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3a599f3c6a64b8ca64c8f44cc437f05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731766592158 2024-11-16T14:16:35,677 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting b8a811abd6df4986a3096202800ed80b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731766592569 2024-11-16T14:16:35,677 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 144f9b34516446779e47c2e2f62c32c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731766594788 2024-11-16T14:16:35,677 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b954ccd6fe4459b92778ee017f95904, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731766592569 2024-11-16T14:16:35,678 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67ef39a320424759935194995e2b3c64, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731766594788 2024-11-16T14:16:35,687 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:35,687 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/8157a91695b44689aac32c5dbe4e4223 is 50, key is test_row_0/B:col10/1731766594788/Put/seqid=0 2024-11-16T14:16:35,699 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#A#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:35,699 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/70621ffeb49048a4a8d5fd2550771e8e is 50, key is test_row_0/A:col10/1731766594788/Put/seqid=0 2024-11-16T14:16:35,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742330_1506 (size=13153) 2024-11-16T14:16:35,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742331_1507 (size=13153) 2024-11-16T14:16:35,714 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/8157a91695b44689aac32c5dbe4e4223 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/8157a91695b44689aac32c5dbe4e4223 2024-11-16T14:16:35,718 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4824a7508eaae82911857430ab75750c/B of 4824a7508eaae82911857430ab75750c into 8157a91695b44689aac32c5dbe4e4223(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:35,718 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:35,718 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/B, priority=12, startTime=1731766595674; duration=0sec 2024-11-16T14:16:35,718 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:35,718 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:B 2024-11-16T14:16:35,719 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:16:35,720 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:16:35,720 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/C is initiating minor compaction (all files) 2024-11-16T14:16:35,720 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/C in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:35,720 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/ce77b5c6b9b64e7b8e0711d95d007187, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/77411ebf37714f9da6b709549b14225d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/7c1ee7f2a14048a1b6d47bb1d912dc1d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/73f1e464146246d7b7b93d7f020b04df] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=46.4 K 2024-11-16T14:16:35,720 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ce77b5c6b9b64e7b8e0711d95d007187, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731766590807 2024-11-16T14:16:35,720 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 77411ebf37714f9da6b709549b14225d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731766592158 2024-11-16T14:16:35,721 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c1ee7f2a14048a1b6d47bb1d912dc1d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731766592569 2024-11-16T14:16:35,721 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 73f1e464146246d7b7b93d7f020b04df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731766594788 2024-11-16T14:16:35,732 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#C#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:35,733 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/0e07bcd4a236450bbd303a882d5f9019 is 50, key is test_row_0/C:col10/1731766594788/Put/seqid=0 2024-11-16T14:16:35,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742332_1508 (size=13153) 2024-11-16T14:16:35,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T14:16:35,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:35,761 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-16T14:16:35,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:35,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:35,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:35,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:35,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:35,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:35,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/8b98b3193ff04b158bb9013631dba739 is 50, key is test_row_0/A:col10/1731766594983/Put/seqid=0 2024-11-16T14:16:35,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742333_1509 (size=12301) 2024-11-16T14:16:35,780 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/8b98b3193ff04b158bb9013631dba739 2024-11-16T14:16:35,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/9cd59b3f2ff5404d94b2d82e7f6ca324 is 50, key is test_row_0/B:col10/1731766594983/Put/seqid=0 2024-11-16T14:16:35,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742334_1510 (size=12301) 2024-11-16T14:16:35,819 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/9cd59b3f2ff5404d94b2d82e7f6ca324 2024-11-16T14:16:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T14:16:35,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/4c2e575c260a454ba2dd3cfb5003dbe1 is 50, key is test_row_0/C:col10/1731766594983/Put/seqid=0 2024-11-16T14:16:35,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742335_1511 (size=12301) 2024-11-16T14:16:36,113 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/70621ffeb49048a4a8d5fd2550771e8e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/70621ffeb49048a4a8d5fd2550771e8e 2024-11-16T14:16:36,119 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4824a7508eaae82911857430ab75750c/A of 4824a7508eaae82911857430ab75750c into 70621ffeb49048a4a8d5fd2550771e8e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:36,119 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:36,119 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/A, priority=12, startTime=1731766595674; duration=0sec 2024-11-16T14:16:36,119 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:36,119 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:A 2024-11-16T14:16:36,146 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/0e07bcd4a236450bbd303a882d5f9019 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0e07bcd4a236450bbd303a882d5f9019 2024-11-16T14:16:36,149 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4824a7508eaae82911857430ab75750c/C of 4824a7508eaae82911857430ab75750c into 0e07bcd4a236450bbd303a882d5f9019(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:36,149 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:36,149 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/C, priority=12, startTime=1731766595674; duration=0sec 2024-11-16T14:16:36,149 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:36,149 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:C 2024-11-16T14:16:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:36,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:36,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766656183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:36,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766656185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:36,235 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/4c2e575c260a454ba2dd3cfb5003dbe1 2024-11-16T14:16:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/8b98b3193ff04b158bb9013631dba739 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/8b98b3193ff04b158bb9013631dba739 2024-11-16T14:16:36,242 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/8b98b3193ff04b158bb9013631dba739, entries=150, sequenceid=380, filesize=12.0 K 2024-11-16T14:16:36,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/9cd59b3f2ff5404d94b2d82e7f6ca324 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9cd59b3f2ff5404d94b2d82e7f6ca324 2024-11-16T14:16:36,246 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9cd59b3f2ff5404d94b2d82e7f6ca324, entries=150, sequenceid=380, filesize=12.0 K 2024-11-16T14:16:36,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/4c2e575c260a454ba2dd3cfb5003dbe1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4c2e575c260a454ba2dd3cfb5003dbe1 2024-11-16T14:16:36,251 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4c2e575c260a454ba2dd3cfb5003dbe1, entries=150, sequenceid=380, filesize=12.0 K 2024-11-16T14:16:36,252 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 4824a7508eaae82911857430ab75750c in 491ms, sequenceid=380, compaction requested=false 2024-11-16T14:16:36,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:36,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:36,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-16T14:16:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-16T14:16:36,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-16T14:16:36,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5660 sec 2024-11-16T14:16:36,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.5700 sec 2024-11-16T14:16:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:36,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:16:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:36,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/5a3d9964c57a4a5cba14dc51fcd3f657 is 50, key is test_row_0/A:col10/1731766596299/Put/seqid=0 2024-11-16T14:16:36,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742336_1512 (size=14741) 2024-11-16T14:16:36,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/5a3d9964c57a4a5cba14dc51fcd3f657 2024-11-16T14:16:36,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/45efd8e858774e01acc88dc274cce011 is 50, key is test_row_0/B:col10/1731766596299/Put/seqid=0 2024-11-16T14:16:36,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742337_1513 (size=12301) 2024-11-16T14:16:36,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/45efd8e858774e01acc88dc274cce011 2024-11-16T14:16:36,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/af9e9069e15d4cedbfe2e8cbbede0217 is 50, key is test_row_0/C:col10/1731766596299/Put/seqid=0 2024-11-16T14:16:36,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766596377, max timestamp=1731766596408Num uniques 859; 2024-11-16T14:16:36,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:16:36,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766596377, max timestamp=1731766596408Num uniques 859; 2024-11-16T14:16:36,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:16:36,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731766596377, max timestamp=1731766596408Num uniques 859; 2024-11-16T14:16:36,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:16:36,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742338_1514 (size=12301) 2024-11-16T14:16:36,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766656485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766656488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:36,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766656605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:36,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:36,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766656605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:36,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:36,721 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:16:36,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/af9e9069e15d4cedbfe2e8cbbede0217 2024-11-16T14:16:36,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/5a3d9964c57a4a5cba14dc51fcd3f657 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/5a3d9964c57a4a5cba14dc51fcd3f657 2024-11-16T14:16:36,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/5a3d9964c57a4a5cba14dc51fcd3f657, entries=200, sequenceid=393, filesize=14.4 K 2024-11-16T14:16:36,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/45efd8e858774e01acc88dc274cce011 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/45efd8e858774e01acc88dc274cce011 2024-11-16T14:16:36,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/45efd8e858774e01acc88dc274cce011, entries=150, sequenceid=393, filesize=12.0 K 2024-11-16T14:16:36,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/af9e9069e15d4cedbfe2e8cbbede0217 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/af9e9069e15d4cedbfe2e8cbbede0217 2024-11-16T14:16:36,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766656816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:36,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766656816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:36,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/af9e9069e15d4cedbfe2e8cbbede0217, entries=150, sequenceid=393, filesize=12.0 K 2024-11-16T14:16:36,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4824a7508eaae82911857430ab75750c in 530ms, sequenceid=393, compaction requested=true 2024-11-16T14:16:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:36,832 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:36,832 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:36,832 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:36,833 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:36,833 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/A is initiating minor compaction (all files) 2024-11-16T14:16:36,833 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/B is initiating minor compaction (all files) 2024-11-16T14:16:36,833 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/A in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:36,833 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/B in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:36,833 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/8157a91695b44689aac32c5dbe4e4223, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9cd59b3f2ff5404d94b2d82e7f6ca324, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/45efd8e858774e01acc88dc274cce011] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=36.9 K 2024-11-16T14:16:36,833 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/70621ffeb49048a4a8d5fd2550771e8e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/8b98b3193ff04b158bb9013631dba739, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/5a3d9964c57a4a5cba14dc51fcd3f657] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=39.3 K 2024-11-16T14:16:36,833 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70621ffeb49048a4a8d5fd2550771e8e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731766594788 2024-11-16T14:16:36,833 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8157a91695b44689aac32c5dbe4e4223, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731766594788 2024-11-16T14:16:36,834 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cd59b3f2ff5404d94b2d82e7f6ca324, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731766594954 2024-11-16T14:16:36,834 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b98b3193ff04b158bb9013631dba739, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731766594954 2024-11-16T14:16:36,834 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a3d9964c57a4a5cba14dc51fcd3f657, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731766596167 2024-11-16T14:16:36,834 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 45efd8e858774e01acc88dc274cce011, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731766596167 2024-11-16T14:16:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T14:16:36,836 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:36,836 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-16T14:16:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T14:16:36,838 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:36,838 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:36,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:36,843 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:36,844 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/f570199d8e9c4b08be16620e9a2812b2 is 50, key is test_row_0/B:col10/1731766596299/Put/seqid=0 2024-11-16T14:16:36,846 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#A#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:36,847 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/448fec3f48cc474b9c2b69e675618d1b is 50, key is test_row_0/A:col10/1731766596299/Put/seqid=0 2024-11-16T14:16:36,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742340_1516 (size=13255) 2024-11-16T14:16:36,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742339_1515 (size=13255) 2024-11-16T14:16:36,866 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/448fec3f48cc474b9c2b69e675618d1b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/448fec3f48cc474b9c2b69e675618d1b 2024-11-16T14:16:36,869 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/A of 4824a7508eaae82911857430ab75750c into 448fec3f48cc474b9c2b69e675618d1b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:36,869 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:36,869 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/A, priority=13, startTime=1731766596832; duration=0sec 2024-11-16T14:16:36,870 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:36,870 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:A 2024-11-16T14:16:36,870 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:36,870 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:36,870 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/C is initiating minor compaction (all files) 2024-11-16T14:16:36,870 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/C in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:36,870 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0e07bcd4a236450bbd303a882d5f9019, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4c2e575c260a454ba2dd3cfb5003dbe1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/af9e9069e15d4cedbfe2e8cbbede0217] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=36.9 K 2024-11-16T14:16:36,871 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e07bcd4a236450bbd303a882d5f9019, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731766594788 2024-11-16T14:16:36,871 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c2e575c260a454ba2dd3cfb5003dbe1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731766594954 2024-11-16T14:16:36,871 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting af9e9069e15d4cedbfe2e8cbbede0217, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731766596167 2024-11-16T14:16:36,879 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:36,880 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/6dc337091d9d4cc5916568708b296e2e is 50, key is test_row_0/C:col10/1731766596299/Put/seqid=0 2024-11-16T14:16:36,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742341_1517 (size=13255) 2024-11-16T14:16:36,904 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/6dc337091d9d4cc5916568708b296e2e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/6dc337091d9d4cc5916568708b296e2e 2024-11-16T14:16:36,911 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/C of 4824a7508eaae82911857430ab75750c into 6dc337091d9d4cc5916568708b296e2e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:36,911 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:36,911 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/C, priority=13, startTime=1731766596832; duration=0sec 2024-11-16T14:16:36,911 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:36,912 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:C 2024-11-16T14:16:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T14:16:36,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T14:16:36,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:36,993 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-16T14:16:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:36,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/556ba6de0f4d452cb92a4fe0af5415b4 is 50, key is test_row_0/A:col10/1731766596479/Put/seqid=0 2024-11-16T14:16:37,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742342_1518 (size=12301) 2024-11-16T14:16:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:37,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T14:16:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766657172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766657173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39360 deadline: 1731766657176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,185 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_2' in 4824a7508eaae82911857430ab75750c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:37,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39426 deadline: 1731766657195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,199 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in 4824a7508eaae82911857430ab75750c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,274 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/f570199d8e9c4b08be16620e9a2812b2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/f570199d8e9c4b08be16620e9a2812b2 2024-11-16T14:16:37,286 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/B of 4824a7508eaae82911857430ab75750c into f570199d8e9c4b08be16620e9a2812b2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:37,286 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:37,286 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/B, priority=13, startTime=1731766596832; duration=0sec 2024-11-16T14:16:37,286 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:37,286 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:B 2024-11-16T14:16:37,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766657279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766657286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,415 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/556ba6de0f4d452cb92a4fe0af5415b4 2024-11-16T14:16:37,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/670665e4dbd74d489fd9424e4867ec6b is 50, key is test_row_0/B:col10/1731766596479/Put/seqid=0 2024-11-16T14:16:37,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742343_1519 (size=12301) 2024-11-16T14:16:37,433 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/670665e4dbd74d489fd9424e4867ec6b 2024-11-16T14:16:37,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/537a870ac2cf4dcf99056a24d270cae6 is 50, key is test_row_0/C:col10/1731766596479/Put/seqid=0 2024-11-16T14:16:37,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742344_1520 (size=12301) 2024-11-16T14:16:37,457 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/537a870ac2cf4dcf99056a24d270cae6 2024-11-16T14:16:37,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/556ba6de0f4d452cb92a4fe0af5415b4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/556ba6de0f4d452cb92a4fe0af5415b4 2024-11-16T14:16:37,465 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/556ba6de0f4d452cb92a4fe0af5415b4, entries=150, sequenceid=430, filesize=12.0 K 2024-11-16T14:16:37,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/670665e4dbd74d489fd9424e4867ec6b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/670665e4dbd74d489fd9424e4867ec6b 2024-11-16T14:16:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T14:16:37,469 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/670665e4dbd74d489fd9424e4867ec6b, entries=150, sequenceid=430, filesize=12.0 K 2024-11-16T14:16:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/537a870ac2cf4dcf99056a24d270cae6 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/537a870ac2cf4dcf99056a24d270cae6 2024-11-16T14:16:37,475 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/537a870ac2cf4dcf99056a24d270cae6, entries=150, sequenceid=430, filesize=12.0 K 2024-11-16T14:16:37,482 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 4824a7508eaae82911857430ab75750c in 489ms, sequenceid=430, compaction requested=false 2024-11-16T14:16:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-16T14:16:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-16T14:16:37,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-16T14:16:37,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 644 msec 2024-11-16T14:16:37,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 652 msec 2024-11-16T14:16:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:37,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:16:37,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:37,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:37,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:37,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:37,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:37,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:37,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/3e30afed7e3144e69c2fa5e50114b5b5 is 50, key is test_row_0/A:col10/1731766597498/Put/seqid=0 2024-11-16T14:16:37,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742345_1521 (size=9857) 2024-11-16T14:16:37,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/3e30afed7e3144e69c2fa5e50114b5b5 2024-11-16T14:16:37,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/3bedbdef20c942b685c0d95b852f7bdf is 50, key is test_row_0/B:col10/1731766597498/Put/seqid=0 2024-11-16T14:16:37,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742346_1522 (size=9857) 2024-11-16T14:16:37,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/3bedbdef20c942b685c0d95b852f7bdf 2024-11-16T14:16:37,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/4d706510f3c84eb79b6f65b3af6f0c9a is 50, key is test_row_0/C:col10/1731766597498/Put/seqid=0 2024-11-16T14:16:37,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742347_1523 (size=9857) 2024-11-16T14:16:37,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/4d706510f3c84eb79b6f65b3af6f0c9a 2024-11-16T14:16:37,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/3e30afed7e3144e69c2fa5e50114b5b5 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3e30afed7e3144e69c2fa5e50114b5b5 2024-11-16T14:16:37,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3e30afed7e3144e69c2fa5e50114b5b5, entries=100, sequenceid=443, filesize=9.6 K 2024-11-16T14:16:37,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/3bedbdef20c942b685c0d95b852f7bdf as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/3bedbdef20c942b685c0d95b852f7bdf 2024-11-16T14:16:37,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/3bedbdef20c942b685c0d95b852f7bdf, entries=100, sequenceid=443, filesize=9.6 K 2024-11-16T14:16:37,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/4d706510f3c84eb79b6f65b3af6f0c9a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4d706510f3c84eb79b6f65b3af6f0c9a 2024-11-16T14:16:37,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4d706510f3c84eb79b6f65b3af6f0c9a, entries=100, sequenceid=443, filesize=9.6 K 2024-11-16T14:16:37,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=93.93 KB/96180 for 4824a7508eaae82911857430ab75750c in 58ms, sequenceid=443, compaction requested=true 2024-11-16T14:16:37,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:37,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:37,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:37,558 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:37,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:37,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:37,558 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:37,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:37,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:37,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T14:16:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:37,560 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:37,560 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:37,561 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/B is initiating minor compaction (all files) 2024-11-16T14:16:37,561 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/A is initiating minor compaction (all files) 2024-11-16T14:16:37,561 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/A in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:37,561 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/B in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:37,561 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/448fec3f48cc474b9c2b69e675618d1b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/556ba6de0f4d452cb92a4fe0af5415b4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3e30afed7e3144e69c2fa5e50114b5b5] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=34.6 K 2024-11-16T14:16:37,561 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/f570199d8e9c4b08be16620e9a2812b2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/670665e4dbd74d489fd9424e4867ec6b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/3bedbdef20c942b685c0d95b852f7bdf] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=34.6 K 2024-11-16T14:16:37,561 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f570199d8e9c4b08be16620e9a2812b2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731766596167 2024-11-16T14:16:37,561 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 448fec3f48cc474b9c2b69e675618d1b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731766596167 2024-11-16T14:16:37,562 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 670665e4dbd74d489fd9424e4867ec6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731766596469 2024-11-16T14:16:37,562 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 556ba6de0f4d452cb92a4fe0af5415b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731766596469 2024-11-16T14:16:37,562 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e30afed7e3144e69c2fa5e50114b5b5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731766597171 2024-11-16T14:16:37,562 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bedbdef20c942b685c0d95b852f7bdf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731766597171 2024-11-16T14:16:37,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/12b2017010414abeaf728c4096773075 is 50, key is test_row_0/A:col10/1731766597557/Put/seqid=0 2024-11-16T14:16:37,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742348_1524 (size=17181) 2024-11-16T14:16:37,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/12b2017010414abeaf728c4096773075 2024-11-16T14:16:37,572 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#B#compaction#454 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:37,572 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/ccc55a2c8f4045d687098a191bb9bf54 is 50, key is test_row_0/B:col10/1731766597498/Put/seqid=0 2024-11-16T14:16:37,574 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#A#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:37,574 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/fadd0e0a4a3e48cc92bd0581e0f40163 is 50, key is test_row_0/A:col10/1731766597498/Put/seqid=0 2024-11-16T14:16:37,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742350_1526 (size=13357) 2024-11-16T14:16:37,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742349_1525 (size=13357) 2024-11-16T14:16:37,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/4ce5d21796984683bcd69bab807ea49c is 50, key is test_row_0/B:col10/1731766597557/Put/seqid=0 2024-11-16T14:16:37,585 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/fadd0e0a4a3e48cc92bd0581e0f40163 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/fadd0e0a4a3e48cc92bd0581e0f40163 2024-11-16T14:16:37,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742351_1527 (size=12301) 2024-11-16T14:16:37,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/4ce5d21796984683bcd69bab807ea49c 2024-11-16T14:16:37,591 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/A of 4824a7508eaae82911857430ab75750c into fadd0e0a4a3e48cc92bd0581e0f40163(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:37,591 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:37,592 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/A, priority=13, startTime=1731766597558; duration=0sec 2024-11-16T14:16:37,592 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:37,592 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:A 2024-11-16T14:16:37,592 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:37,600 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:37,600 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/C is initiating minor compaction (all files) 2024-11-16T14:16:37,600 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/C in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:37,601 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/6dc337091d9d4cc5916568708b296e2e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/537a870ac2cf4dcf99056a24d270cae6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4d706510f3c84eb79b6f65b3af6f0c9a] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=34.6 K 2024-11-16T14:16:37,604 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dc337091d9d4cc5916568708b296e2e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731766596167 2024-11-16T14:16:37,604 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 537a870ac2cf4dcf99056a24d270cae6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731766596469 2024-11-16T14:16:37,605 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d706510f3c84eb79b6f65b3af6f0c9a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731766597171 2024-11-16T14:16:37,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/23324fd6ac66440394ebec49e6069a64 is 50, key is test_row_0/C:col10/1731766597557/Put/seqid=0 2024-11-16T14:16:37,614 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:37,614 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/ba908bf5eb2e45ce9d8d55b4e4340e0f is 50, key is test_row_0/C:col10/1731766597498/Put/seqid=0 2024-11-16T14:16:37,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742352_1528 (size=13357) 2024-11-16T14:16:37,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742353_1529 (size=12301) 2024-11-16T14:16:37,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766657629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766657637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766657745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766657755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39412 deadline: 1731766657916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,923 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_1' in 4824a7508eaae82911857430ab75750c of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18352 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766657955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766657965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T14:16:37,976 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:37,976 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-16T14:16:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:16:37,977 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:37,978 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:37,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:37,985 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/ccc55a2c8f4045d687098a191bb9bf54 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/ccc55a2c8f4045d687098a191bb9bf54 2024-11-16T14:16:37,989 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/B of 4824a7508eaae82911857430ab75750c into ccc55a2c8f4045d687098a191bb9bf54(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:37,989 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:37,989 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/B, priority=13, startTime=1731766597558; duration=0sec 2024-11-16T14:16:37,989 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:37,989 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:B 2024-11-16T14:16:38,035 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/ba908bf5eb2e45ce9d8d55b4e4340e0f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/ba908bf5eb2e45ce9d8d55b4e4340e0f 2024-11-16T14:16:38,039 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4824a7508eaae82911857430ab75750c/C of 4824a7508eaae82911857430ab75750c into ba908bf5eb2e45ce9d8d55b4e4340e0f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:38,039 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:38,039 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/C, priority=13, startTime=1731766597558; duration=0sec 2024-11-16T14:16:38,039 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:38,039 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:C 2024-11-16T14:16:38,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/23324fd6ac66440394ebec49e6069a64 2024-11-16T14:16:38,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/12b2017010414abeaf728c4096773075 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/12b2017010414abeaf728c4096773075 2024-11-16T14:16:38,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/12b2017010414abeaf728c4096773075, entries=250, sequenceid=461, filesize=16.8 K 2024-11-16T14:16:38,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/4ce5d21796984683bcd69bab807ea49c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/4ce5d21796984683bcd69bab807ea49c 2024-11-16T14:16:38,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/4ce5d21796984683bcd69bab807ea49c, entries=150, sequenceid=461, filesize=12.0 K 2024-11-16T14:16:38,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/23324fd6ac66440394ebec49e6069a64 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/23324fd6ac66440394ebec49e6069a64 2024-11-16T14:16:38,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/23324fd6ac66440394ebec49e6069a64, entries=150, sequenceid=461, filesize=12.0 K 2024-11-16T14:16:38,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4824a7508eaae82911857430ab75750c in 498ms, sequenceid=461, compaction requested=false 2024-11-16T14:16:38,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:16:38,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T14:16:38,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:38,130 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T14:16:38,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:38,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:38,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:38,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:38,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:38,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:38,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/21adeac6a9ca444dafbf6ed36b8937e3 is 50, key is test_row_0/A:col10/1731766597632/Put/seqid=0 2024-11-16T14:16:38,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742354_1530 (size=12301) 2024-11-16T14:16:38,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:38,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. as already flushing 2024-11-16T14:16:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:16:38,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766658323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:38,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766658328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:38,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766658436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:38,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766658445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:38,548 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/21adeac6a9ca444dafbf6ed36b8937e3 2024-11-16T14:16:38,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/4e1876f990e8471ba11e10c6c9ee2e3c is 50, key is test_row_0/B:col10/1731766597632/Put/seqid=0 2024-11-16T14:16:38,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742355_1531 (size=12301) 2024-11-16T14:16:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:16:38,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766658646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:38,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766658656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:38,958 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/4e1876f990e8471ba11e10c6c9ee2e3c 2024-11-16T14:16:38,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/264cf405bcc442e49011a7e442859126 is 50, key is test_row_0/C:col10/1731766597632/Put/seqid=0 2024-11-16T14:16:38,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742356_1532 (size=12301) 2024-11-16T14:16:38,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:38,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731766658965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:38,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1731766658975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 , the old value is region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., hostname=04e93f2b5fad,41317,1731766448342, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:16:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:16:39,366 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/264cf405bcc442e49011a7e442859126 2024-11-16T14:16:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/21adeac6a9ca444dafbf6ed36b8937e3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/21adeac6a9ca444dafbf6ed36b8937e3 2024-11-16T14:16:39,372 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/21adeac6a9ca444dafbf6ed36b8937e3, entries=150, sequenceid=478, filesize=12.0 K 2024-11-16T14:16:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/4e1876f990e8471ba11e10c6c9ee2e3c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/4e1876f990e8471ba11e10c6c9ee2e3c 2024-11-16T14:16:39,375 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/4e1876f990e8471ba11e10c6c9ee2e3c, entries=150, sequenceid=478, filesize=12.0 K 2024-11-16T14:16:39,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/264cf405bcc442e49011a7e442859126 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/264cf405bcc442e49011a7e442859126 2024-11-16T14:16:39,378 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/264cf405bcc442e49011a7e442859126, entries=150, sequenceid=478, filesize=12.0 K 2024-11-16T14:16:39,379 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4824a7508eaae82911857430ab75750c in 1249ms, sequenceid=478, compaction requested=true 2024-11-16T14:16:39,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:39,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:39,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-16T14:16:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-16T14:16:39,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-16T14:16:39,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4020 sec 2024-11-16T14:16:39,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.4070 sec 2024-11-16T14:16:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:39,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:16:39,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:39,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:39,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:39,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:39,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:39,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:39,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/6953e5a50ccf4bdda57091a72ef15f9b is 50, key is test_row_0/A:col10/1731766599491/Put/seqid=0 2024-11-16T14:16:39,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742357_1533 (size=14741) 2024-11-16T14:16:39,518 INFO [Thread-1960 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1960. 2024-11-16T14:16:39,518 DEBUG [Thread-1960 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:39,518 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,518 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:39,529 INFO [Thread-1958 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1958. 2024-11-16T14:16:39,529 DEBUG [Thread-1958 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:39,529 DEBUG [Thread-1958 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,529 DEBUG [Thread-1958 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:39,534 INFO [Thread-1951 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1951. 2024-11-16T14:16:39,534 DEBUG [Thread-1951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:39,534 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,534 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-11-16T14:16:39,534 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:39,534 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,534 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:39,534 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:39,538 INFO [Thread-1962 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1962. 2024-11-16T14:16:39,538 DEBUG [Thread-1962 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:39,538 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,539 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,539 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:39,545 INFO [Thread-1954 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1954. 2024-11-16T14:16:39,545 DEBUG [Thread-1954 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:39,545 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,546 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:39,550 INFO [Thread-1956 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1956. 2024-11-16T14:16:39,550 DEBUG [Thread-1956 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:39,550 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,550 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:39,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:39,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/6953e5a50ccf4bdda57091a72ef15f9b 2024-11-16T14:16:39,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/72716fa7fb504d4a9c7dd16cff73374b is 50, key is test_row_0/B:col10/1731766599491/Put/seqid=0 2024-11-16T14:16:39,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742358_1534 (size=12301) 2024-11-16T14:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:16:40,127 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:40,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/72716fa7fb504d4a9c7dd16cff73374b 2024-11-16T14:16:40,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/5a34858c499948288b24e44fa553cb26 is 50, key is test_row_0/C:col10/1731766599491/Put/seqid=0 2024-11-16T14:16:40,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742359_1535 (size=12301) 2024-11-16T14:16:40,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/5a34858c499948288b24e44fa553cb26 2024-11-16T14:16:40,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/6953e5a50ccf4bdda57091a72ef15f9b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/6953e5a50ccf4bdda57091a72ef15f9b 2024-11-16T14:16:40,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/6953e5a50ccf4bdda57091a72ef15f9b, entries=200, sequenceid=497, filesize=14.4 K 2024-11-16T14:16:40,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/72716fa7fb504d4a9c7dd16cff73374b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/72716fa7fb504d4a9c7dd16cff73374b 2024-11-16T14:16:40,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/72716fa7fb504d4a9c7dd16cff73374b, entries=150, sequenceid=497, filesize=12.0 K 2024-11-16T14:16:40,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/5a34858c499948288b24e44fa553cb26 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5a34858c499948288b24e44fa553cb26 2024-11-16T14:16:40,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5a34858c499948288b24e44fa553cb26, entries=150, sequenceid=497, filesize=12.0 K 2024-11-16T14:16:40,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=40.25 KB/41220 for 4824a7508eaae82911857430ab75750c in 1252ms, sequenceid=497, compaction requested=true 2024-11-16T14:16:40,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:40,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:40,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:40,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:40,745 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:16:40,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:40,745 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:16:40,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4824a7508eaae82911857430ab75750c:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:40,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:40,747 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:16:40,747 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:16:40,747 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/A is initiating minor compaction (all files) 2024-11-16T14:16:40,747 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/B is initiating minor compaction (all files) 2024-11-16T14:16:40,747 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/B in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:40,747 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/A in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:40,747 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/ccc55a2c8f4045d687098a191bb9bf54, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/4ce5d21796984683bcd69bab807ea49c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/4e1876f990e8471ba11e10c6c9ee2e3c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/72716fa7fb504d4a9c7dd16cff73374b] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=49.1 K 2024-11-16T14:16:40,747 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/fadd0e0a4a3e48cc92bd0581e0f40163, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/12b2017010414abeaf728c4096773075, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/21adeac6a9ca444dafbf6ed36b8937e3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/6953e5a50ccf4bdda57091a72ef15f9b] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=56.2 K 2024-11-16T14:16:40,748 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting fadd0e0a4a3e48cc92bd0581e0f40163, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731766596486 2024-11-16T14:16:40,748 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ccc55a2c8f4045d687098a191bb9bf54, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731766596486 2024-11-16T14:16:40,748 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12b2017010414abeaf728c4096773075, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731766597501 2024-11-16T14:16:40,748 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce5d21796984683bcd69bab807ea49c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731766597549 2024-11-16T14:16:40,748 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21adeac6a9ca444dafbf6ed36b8937e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731766597602 2024-11-16T14:16:40,748 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e1876f990e8471ba11e10c6c9ee2e3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731766597602 2024-11-16T14:16:40,749 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6953e5a50ccf4bdda57091a72ef15f9b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731766598315 2024-11-16T14:16:40,749 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 72716fa7fb504d4a9c7dd16cff73374b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731766598321 2024-11-16T14:16:40,754 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:40,755 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/d82a2b5fed2740ed9a619ca959400612 is 50, key is test_row_0/B:col10/1731766599491/Put/seqid=0 2024-11-16T14:16:40,755 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#A#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:40,756 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/2d9f2bc0b50c42f4a50b512b5e676b5e is 50, key is test_row_0/A:col10/1731766599491/Put/seqid=0 2024-11-16T14:16:40,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742360_1536 (size=13493) 2024-11-16T14:16:40,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742361_1537 (size=13493) 2024-11-16T14:16:41,167 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/d82a2b5fed2740ed9a619ca959400612 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/d82a2b5fed2740ed9a619ca959400612 2024-11-16T14:16:41,167 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/2d9f2bc0b50c42f4a50b512b5e676b5e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/2d9f2bc0b50c42f4a50b512b5e676b5e 2024-11-16T14:16:41,173 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4824a7508eaae82911857430ab75750c/A of 4824a7508eaae82911857430ab75750c into 2d9f2bc0b50c42f4a50b512b5e676b5e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:41,173 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4824a7508eaae82911857430ab75750c/B of 4824a7508eaae82911857430ab75750c into d82a2b5fed2740ed9a619ca959400612(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:41,173 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:41,173 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:41,173 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/A, priority=12, startTime=1731766600745; duration=0sec 2024-11-16T14:16:41,173 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/B, priority=12, startTime=1731766600745; duration=0sec 2024-11-16T14:16:41,173 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:41,173 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:A 2024-11-16T14:16:41,173 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:41,173 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:B 2024-11-16T14:16:41,173 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:16:41,174 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:16:41,174 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): 4824a7508eaae82911857430ab75750c/C is initiating minor compaction (all files) 2024-11-16T14:16:41,174 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4824a7508eaae82911857430ab75750c/C in TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:41,174 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/ba908bf5eb2e45ce9d8d55b4e4340e0f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/23324fd6ac66440394ebec49e6069a64, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/264cf405bcc442e49011a7e442859126, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5a34858c499948288b24e44fa553cb26] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp, totalSize=49.1 K 2024-11-16T14:16:41,174 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba908bf5eb2e45ce9d8d55b4e4340e0f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731766596486 2024-11-16T14:16:41,175 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23324fd6ac66440394ebec49e6069a64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731766597549 2024-11-16T14:16:41,175 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 264cf405bcc442e49011a7e442859126, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731766597602 2024-11-16T14:16:41,175 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a34858c499948288b24e44fa553cb26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731766598321 2024-11-16T14:16:41,180 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4824a7508eaae82911857430ab75750c#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:41,181 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/f4690aa48f9a4bfebb7feda42bbe2388 is 50, key is test_row_0/C:col10/1731766599491/Put/seqid=0 2024-11-16T14:16:41,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742362_1538 (size=13493) 2024-11-16T14:16:41,592 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/f4690aa48f9a4bfebb7feda42bbe2388 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/f4690aa48f9a4bfebb7feda42bbe2388 2024-11-16T14:16:41,600 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4824a7508eaae82911857430ab75750c/C of 4824a7508eaae82911857430ab75750c into f4690aa48f9a4bfebb7feda42bbe2388(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:41,600 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:41,600 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., storeName=4824a7508eaae82911857430ab75750c/C, priority=12, startTime=1731766600745; duration=0sec 2024-11-16T14:16:41,600 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:41,600 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4824a7508eaae82911857430ab75750c:C 2024-11-16T14:16:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:47,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:16:47,218 INFO [Thread-1949 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1949. 2024-11-16T14:16:47,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:47,218 DEBUG [Thread-1949 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:47,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:47,218 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:47,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:47,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:47,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:47,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:47,219 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:47,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:47,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/db0f9ee1782449a285dfb171ebf34435 is 50, key is test_row_0/A:col10/1731766599503/Put/seqid=0 2024-11-16T14:16:47,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742363_1539 (size=12301) 2024-11-16T14:16:47,308 INFO [Thread-1947 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1947. 2024-11-16T14:16:47,308 DEBUG [Thread-1947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:47,308 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:47,308 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:47,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:47,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/db0f9ee1782449a285dfb171ebf34435 2024-11-16T14:16:47,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/9591ac8ff78e4f27b71b6a1db4e2d244 is 50, key is test_row_0/B:col10/1731766599503/Put/seqid=0 2024-11-16T14:16:47,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742364_1540 (size=12301) 2024-11-16T14:16:48,019 INFO [Thread-1943 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1943. 2024-11-16T14:16:48,019 DEBUG [Thread-1943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:16:48,019 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:48,019 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:48,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:48,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T14:16:48,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-16T14:16:48,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 154 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1174 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3522 rows 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1176 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3520 rows 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1174 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3520 rows 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1167 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3499 rows 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1170 2024-11-16T14:16:48,020 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3508 rows 2024-11-16T14:16:48,020 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:16:48,020 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:16:48,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:48,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:48,021 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:16:48,022 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T14:16:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T14:16:48,028 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766608027"}]},"ts":"1731766608027"} 2024-11-16T14:16:48,030 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T14:16:48,030 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T14:16:48,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:16:48,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4824a7508eaae82911857430ab75750c, UNASSIGN}] 2024-11-16T14:16:48,035 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4824a7508eaae82911857430ab75750c, UNASSIGN 2024-11-16T14:16:48,036 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=4824a7508eaae82911857430ab75750c, regionState=CLOSING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:48,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4824a7508eaae82911857430ab75750c, UNASSIGN because future has completed 2024-11-16T14:16:48,037 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:16:48,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:16:48,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/9591ac8ff78e4f27b71b6a1db4e2d244 2024-11-16T14:16:48,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/2e6e7feadfbe48e9ba5050abfb6d2cd9 is 50, key is test_row_0/C:col10/1731766599503/Put/seqid=0 2024-11-16T14:16:48,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742365_1541 (size=12301) 2024-11-16T14:16:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T14:16:48,190 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:48,190 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:16:48,190 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 4824a7508eaae82911857430ab75750c, disabling compactions & flushes 2024-11-16T14:16:48,190 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T14:16:48,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/2e6e7feadfbe48e9ba5050abfb6d2cd9 2024-11-16T14:16:48,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/db0f9ee1782449a285dfb171ebf34435 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/db0f9ee1782449a285dfb171ebf34435 2024-11-16T14:16:48,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/db0f9ee1782449a285dfb171ebf34435, entries=150, sequenceid=510, filesize=12.0 K 2024-11-16T14:16:48,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/9591ac8ff78e4f27b71b6a1db4e2d244 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9591ac8ff78e4f27b71b6a1db4e2d244 2024-11-16T14:16:48,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9591ac8ff78e4f27b71b6a1db4e2d244, entries=150, sequenceid=510, filesize=12.0 K 2024-11-16T14:16:48,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/2e6e7feadfbe48e9ba5050abfb6d2cd9 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/2e6e7feadfbe48e9ba5050abfb6d2cd9 2024-11-16T14:16:48,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/2e6e7feadfbe48e9ba5050abfb6d2cd9, entries=150, sequenceid=510, filesize=12.0 K 2024-11-16T14:16:48,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for 4824a7508eaae82911857430ab75750c in 1269ms, sequenceid=510, compaction requested=false 2024-11-16T14:16:48,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4824a7508eaae82911857430ab75750c: 2024-11-16T14:16:48,487 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:48,487 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:48,487 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. after waiting 0 ms 2024-11-16T14:16:48,487 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:48,487 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 4824a7508eaae82911857430ab75750c 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-16T14:16:48,488 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=A 2024-11-16T14:16:48,488 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:48,488 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=B 2024-11-16T14:16:48,488 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:48,488 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4824a7508eaae82911857430ab75750c, store=C 2024-11-16T14:16:48,488 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:48,491 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/682278e5fa6c4fe0b42c32795f4d6794 is 50, key is test_row_0/A:col10/1731766607306/Put/seqid=0 2024-11-16T14:16:48,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742366_1542 (size=9857) 2024-11-16T14:16:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T14:16:48,896 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/682278e5fa6c4fe0b42c32795f4d6794 2024-11-16T14:16:48,903 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/c7217dbb6d6940d788e3fa8ab9c805c6 is 50, key is test_row_0/B:col10/1731766607306/Put/seqid=0 2024-11-16T14:16:48,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742367_1543 (size=9857) 2024-11-16T14:16:49,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T14:16:49,308 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/c7217dbb6d6940d788e3fa8ab9c805c6 2024-11-16T14:16:49,316 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/0351714762bf45cbb17ef1b4c8844ed3 is 50, key is test_row_0/C:col10/1731766607306/Put/seqid=0 2024-11-16T14:16:49,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742368_1544 (size=9857) 2024-11-16T14:16:49,720 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/0351714762bf45cbb17ef1b4c8844ed3 2024-11-16T14:16:49,723 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/A/682278e5fa6c4fe0b42c32795f4d6794 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/682278e5fa6c4fe0b42c32795f4d6794 2024-11-16T14:16:49,725 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/682278e5fa6c4fe0b42c32795f4d6794, entries=100, sequenceid=515, filesize=9.6 K 2024-11-16T14:16:49,726 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/B/c7217dbb6d6940d788e3fa8ab9c805c6 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c7217dbb6d6940d788e3fa8ab9c805c6 2024-11-16T14:16:49,728 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c7217dbb6d6940d788e3fa8ab9c805c6, entries=100, sequenceid=515, filesize=9.6 K 2024-11-16T14:16:49,728 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/.tmp/C/0351714762bf45cbb17ef1b4c8844ed3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0351714762bf45cbb17ef1b4c8844ed3 2024-11-16T14:16:49,730 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0351714762bf45cbb17ef1b4c8844ed3, entries=100, sequenceid=515, filesize=9.6 K 2024-11-16T14:16:49,731 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 4824a7508eaae82911857430ab75750c in 1244ms, sequenceid=515, compaction requested=true 2024-11-16T14:16:49,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/e91b095a689d48f9b02d3ffdcb683b17, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/31d34c591f5a4c6fab19932e99413eab, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/56fb3866469c4641a166fc893f2eae83, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/f7f99ad32949429ab204d61cd69bd9b0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/bce8c8f75ce44c26a7f70dc19c9adad4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/dc69acb3f8744c9fbf0368d1b00b2cf7, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/358d352e476746c1ace1927cbd4bfb46, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/f38715c10f984818bce1c74fe60b0942, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/c44b1f7a40014917999ac872d3607cf0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/23416f3859f94991b66468f1ef40f563, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/0f853da2edb3422b974706c4621a4912, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/b26b0dc52d604665880a7ca1c629d6e9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/013c8981dd484fc6a366c7952fae3c60, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/221980f5960d41b2a74d8dbf53083a96, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/847653016ad84ee29033de50a2891af1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/d93c520bfbf6473f85f667c053aff23a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/ba5b057cfaf04eb18d0522a19a2b08cd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/fe051e4808d84752a77818d4bba068b8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3311f601f44743f38b4b2fbc805c7167, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/e1ee48ffb0ff4fb09d5724aceb94d43f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3e8cbc53f7a249069ca12c8b5575698c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/dc572925d99242b18d3dfc5d7b94e39f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/d3a599f3c6a64b8ca64c8f44cc437f05, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/8b954ccd6fe4459b92778ee017f95904, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/67ef39a320424759935194995e2b3c64, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/70621ffeb49048a4a8d5fd2550771e8e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/8b98b3193ff04b158bb9013631dba739, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/5a3d9964c57a4a5cba14dc51fcd3f657, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/448fec3f48cc474b9c2b69e675618d1b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/556ba6de0f4d452cb92a4fe0af5415b4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/fadd0e0a4a3e48cc92bd0581e0f40163, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3e30afed7e3144e69c2fa5e50114b5b5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/12b2017010414abeaf728c4096773075, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/21adeac6a9ca444dafbf6ed36b8937e3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/6953e5a50ccf4bdda57091a72ef15f9b] to archive 2024-11-16T14:16:49,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:16:49,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/e91b095a689d48f9b02d3ffdcb683b17 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/e91b095a689d48f9b02d3ffdcb683b17 2024-11-16T14:16:49,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/31d34c591f5a4c6fab19932e99413eab to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/31d34c591f5a4c6fab19932e99413eab 2024-11-16T14:16:49,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/56fb3866469c4641a166fc893f2eae83 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/56fb3866469c4641a166fc893f2eae83 2024-11-16T14:16:49,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/f7f99ad32949429ab204d61cd69bd9b0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/f7f99ad32949429ab204d61cd69bd9b0 2024-11-16T14:16:49,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/bce8c8f75ce44c26a7f70dc19c9adad4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/bce8c8f75ce44c26a7f70dc19c9adad4 2024-11-16T14:16:49,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/dc69acb3f8744c9fbf0368d1b00b2cf7 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/dc69acb3f8744c9fbf0368d1b00b2cf7 2024-11-16T14:16:49,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/358d352e476746c1ace1927cbd4bfb46 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/358d352e476746c1ace1927cbd4bfb46 2024-11-16T14:16:49,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/f38715c10f984818bce1c74fe60b0942 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/f38715c10f984818bce1c74fe60b0942 2024-11-16T14:16:49,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/c44b1f7a40014917999ac872d3607cf0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/c44b1f7a40014917999ac872d3607cf0 2024-11-16T14:16:49,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/23416f3859f94991b66468f1ef40f563 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/23416f3859f94991b66468f1ef40f563 2024-11-16T14:16:49,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/0f853da2edb3422b974706c4621a4912 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/0f853da2edb3422b974706c4621a4912 2024-11-16T14:16:49,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/b26b0dc52d604665880a7ca1c629d6e9 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/b26b0dc52d604665880a7ca1c629d6e9 2024-11-16T14:16:49,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/013c8981dd484fc6a366c7952fae3c60 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/013c8981dd484fc6a366c7952fae3c60 2024-11-16T14:16:49,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/221980f5960d41b2a74d8dbf53083a96 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/221980f5960d41b2a74d8dbf53083a96 2024-11-16T14:16:49,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/847653016ad84ee29033de50a2891af1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/847653016ad84ee29033de50a2891af1 2024-11-16T14:16:49,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/d93c520bfbf6473f85f667c053aff23a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/d93c520bfbf6473f85f667c053aff23a 2024-11-16T14:16:49,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/ba5b057cfaf04eb18d0522a19a2b08cd to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/ba5b057cfaf04eb18d0522a19a2b08cd 2024-11-16T14:16:49,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/fe051e4808d84752a77818d4bba068b8 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/fe051e4808d84752a77818d4bba068b8 2024-11-16T14:16:49,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3311f601f44743f38b4b2fbc805c7167 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3311f601f44743f38b4b2fbc805c7167 2024-11-16T14:16:49,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/e1ee48ffb0ff4fb09d5724aceb94d43f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/e1ee48ffb0ff4fb09d5724aceb94d43f 2024-11-16T14:16:49,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3e8cbc53f7a249069ca12c8b5575698c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3e8cbc53f7a249069ca12c8b5575698c 2024-11-16T14:16:49,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/dc572925d99242b18d3dfc5d7b94e39f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/dc572925d99242b18d3dfc5d7b94e39f 2024-11-16T14:16:49,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/d3a599f3c6a64b8ca64c8f44cc437f05 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/d3a599f3c6a64b8ca64c8f44cc437f05 2024-11-16T14:16:49,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/8b954ccd6fe4459b92778ee017f95904 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/8b954ccd6fe4459b92778ee017f95904 2024-11-16T14:16:49,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/67ef39a320424759935194995e2b3c64 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/67ef39a320424759935194995e2b3c64 2024-11-16T14:16:49,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/70621ffeb49048a4a8d5fd2550771e8e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/70621ffeb49048a4a8d5fd2550771e8e 2024-11-16T14:16:49,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/8b98b3193ff04b158bb9013631dba739 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/8b98b3193ff04b158bb9013631dba739 2024-11-16T14:16:49,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/5a3d9964c57a4a5cba14dc51fcd3f657 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/5a3d9964c57a4a5cba14dc51fcd3f657 2024-11-16T14:16:49,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/448fec3f48cc474b9c2b69e675618d1b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/448fec3f48cc474b9c2b69e675618d1b 2024-11-16T14:16:49,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/556ba6de0f4d452cb92a4fe0af5415b4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/556ba6de0f4d452cb92a4fe0af5415b4 2024-11-16T14:16:49,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/fadd0e0a4a3e48cc92bd0581e0f40163 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/fadd0e0a4a3e48cc92bd0581e0f40163 2024-11-16T14:16:49,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3e30afed7e3144e69c2fa5e50114b5b5 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/3e30afed7e3144e69c2fa5e50114b5b5 2024-11-16T14:16:49,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/12b2017010414abeaf728c4096773075 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/12b2017010414abeaf728c4096773075 2024-11-16T14:16:49,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/21adeac6a9ca444dafbf6ed36b8937e3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/21adeac6a9ca444dafbf6ed36b8937e3 2024-11-16T14:16:49,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/6953e5a50ccf4bdda57091a72ef15f9b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/6953e5a50ccf4bdda57091a72ef15f9b 2024-11-16T14:16:49,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/944aa40ae8c74b14bae0babebe1fd157, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c3b5b0cf475147fca8d36f355dc72053, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/781459e5c129415684ac40aeb3adf855, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/220ccc654941402e9c948d3977d91959, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/1c7f8b5e31f041759eceecef004632be, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/72ab0a4694b34f389a41f3998b364dd0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/35bfe85b4713483281aa8890222cc94f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/ca89df5d4370400ab1a21973db3ed27e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/452d27de48c640be8b24c858deb0a8de, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/46afb44a3cf745eab4accb8ab318c547, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c141b8a39e8442f3b027071c2cf648a6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/5d36a61d8fc04e7fb0b1cb9a9042e0d0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/dda52b6aeb65467b812363318ba22204, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9088ef7c72f145429a1a09cf1969a0c6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/622a0594a0894943a044215970dd76a9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b26b24dd9d6a401f9c4d1666449d6635, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/f1294a761fb345e4aae656028215b40c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/3af28a2982364482982c367f03ec4057, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/d4f5efe847d94c58bc7a3a112b87d777, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/16a288940c08465799804560f802b676, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/060ada14e1484da3881450a5ba98f487, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/caae545d9f244ecd86c2ccdf843f3feb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b12de2125fd1485cbd26b3db1c79bdd4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b8a811abd6df4986a3096202800ed80b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/8157a91695b44689aac32c5dbe4e4223, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/144f9b34516446779e47c2e2f62c32c1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9cd59b3f2ff5404d94b2d82e7f6ca324, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/f570199d8e9c4b08be16620e9a2812b2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/45efd8e858774e01acc88dc274cce011, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/670665e4dbd74d489fd9424e4867ec6b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/ccc55a2c8f4045d687098a191bb9bf54, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/3bedbdef20c942b685c0d95b852f7bdf, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/4ce5d21796984683bcd69bab807ea49c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/4e1876f990e8471ba11e10c6c9ee2e3c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/72716fa7fb504d4a9c7dd16cff73374b] to archive 2024-11-16T14:16:49,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:16:49,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/944aa40ae8c74b14bae0babebe1fd157 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/944aa40ae8c74b14bae0babebe1fd157 2024-11-16T14:16:49,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c3b5b0cf475147fca8d36f355dc72053 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c3b5b0cf475147fca8d36f355dc72053 2024-11-16T14:16:49,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/781459e5c129415684ac40aeb3adf855 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/781459e5c129415684ac40aeb3adf855 2024-11-16T14:16:49,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/220ccc654941402e9c948d3977d91959 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/220ccc654941402e9c948d3977d91959 2024-11-16T14:16:49,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/1c7f8b5e31f041759eceecef004632be to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/1c7f8b5e31f041759eceecef004632be 2024-11-16T14:16:49,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/72ab0a4694b34f389a41f3998b364dd0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/72ab0a4694b34f389a41f3998b364dd0 2024-11-16T14:16:49,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/35bfe85b4713483281aa8890222cc94f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/35bfe85b4713483281aa8890222cc94f 2024-11-16T14:16:49,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/ca89df5d4370400ab1a21973db3ed27e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/ca89df5d4370400ab1a21973db3ed27e 2024-11-16T14:16:49,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/452d27de48c640be8b24c858deb0a8de to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/452d27de48c640be8b24c858deb0a8de 2024-11-16T14:16:49,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/46afb44a3cf745eab4accb8ab318c547 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/46afb44a3cf745eab4accb8ab318c547 2024-11-16T14:16:49,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c141b8a39e8442f3b027071c2cf648a6 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c141b8a39e8442f3b027071c2cf648a6 2024-11-16T14:16:49,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/5d36a61d8fc04e7fb0b1cb9a9042e0d0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/5d36a61d8fc04e7fb0b1cb9a9042e0d0 2024-11-16T14:16:49,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/dda52b6aeb65467b812363318ba22204 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/dda52b6aeb65467b812363318ba22204 2024-11-16T14:16:49,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9088ef7c72f145429a1a09cf1969a0c6 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9088ef7c72f145429a1a09cf1969a0c6 2024-11-16T14:16:49,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/622a0594a0894943a044215970dd76a9 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/622a0594a0894943a044215970dd76a9 2024-11-16T14:16:49,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b26b24dd9d6a401f9c4d1666449d6635 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b26b24dd9d6a401f9c4d1666449d6635 2024-11-16T14:16:49,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/f1294a761fb345e4aae656028215b40c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/f1294a761fb345e4aae656028215b40c 2024-11-16T14:16:49,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/3af28a2982364482982c367f03ec4057 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/3af28a2982364482982c367f03ec4057 2024-11-16T14:16:49,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/d4f5efe847d94c58bc7a3a112b87d777 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/d4f5efe847d94c58bc7a3a112b87d777 2024-11-16T14:16:49,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/16a288940c08465799804560f802b676 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/16a288940c08465799804560f802b676 2024-11-16T14:16:49,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/060ada14e1484da3881450a5ba98f487 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/060ada14e1484da3881450a5ba98f487 2024-11-16T14:16:49,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/caae545d9f244ecd86c2ccdf843f3feb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/caae545d9f244ecd86c2ccdf843f3feb 2024-11-16T14:16:49,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b12de2125fd1485cbd26b3db1c79bdd4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b12de2125fd1485cbd26b3db1c79bdd4 2024-11-16T14:16:49,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b8a811abd6df4986a3096202800ed80b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/b8a811abd6df4986a3096202800ed80b 2024-11-16T14:16:49,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/8157a91695b44689aac32c5dbe4e4223 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/8157a91695b44689aac32c5dbe4e4223 2024-11-16T14:16:49,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/144f9b34516446779e47c2e2f62c32c1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/144f9b34516446779e47c2e2f62c32c1 2024-11-16T14:16:49,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9cd59b3f2ff5404d94b2d82e7f6ca324 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9cd59b3f2ff5404d94b2d82e7f6ca324 2024-11-16T14:16:49,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/f570199d8e9c4b08be16620e9a2812b2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/f570199d8e9c4b08be16620e9a2812b2 2024-11-16T14:16:49,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/45efd8e858774e01acc88dc274cce011 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/45efd8e858774e01acc88dc274cce011 2024-11-16T14:16:49,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/670665e4dbd74d489fd9424e4867ec6b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/670665e4dbd74d489fd9424e4867ec6b 2024-11-16T14:16:49,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/ccc55a2c8f4045d687098a191bb9bf54 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/ccc55a2c8f4045d687098a191bb9bf54 2024-11-16T14:16:49,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/3bedbdef20c942b685c0d95b852f7bdf to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/3bedbdef20c942b685c0d95b852f7bdf 2024-11-16T14:16:49,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/4ce5d21796984683bcd69bab807ea49c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/4ce5d21796984683bcd69bab807ea49c 2024-11-16T14:16:49,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/4e1876f990e8471ba11e10c6c9ee2e3c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/4e1876f990e8471ba11e10c6c9ee2e3c 2024-11-16T14:16:49,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/72716fa7fb504d4a9c7dd16cff73374b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/72716fa7fb504d4a9c7dd16cff73374b 2024-11-16T14:16:49,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/196adfa8c7884ebd908987600bcc21a3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/1d7bcfc8126c4e3bad92e11612bc5b21, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/9419a1ff6df647ec87f18ffaf689c43c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c75124e7654e4c90a491ab3c09241476, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/a93cdc38fafb4572b3e656ae9878bb8c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/956169663db54b80987c279c7d4b7087, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/8aa42a0a48b944c498cb99922c1de37a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/10c6f0d941fb40548c1ecef1b841c9f1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0d665e3ddb0c4860b11e0c15b2530e92, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/cab720838001492bb32cfb3c176c4b2c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/08480430bb2644d186595a01c94e31f6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5bd449c0d9044282b76dffbf12df5ee2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c3339c39bcab46b5a717b38bcd544c2b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/e6a1b547001a4a88a47af958bb3481ad, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/b4809c8ac5a14d4281c6dc095b3ed77d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/b4c46ed48880499da696c457bcbfc09e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/6cade305c68b4ace93c863ee3e789ef9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4e6689c444d8494e95e632f8c9eec221, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/1782bf2079ed41908409235e0900c922, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c60410941ae44ff8bb9eef17f6642e1d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/ce77b5c6b9b64e7b8e0711d95d007187, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5da89011ffc8438d953fcd246313678a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/77411ebf37714f9da6b709549b14225d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/7c1ee7f2a14048a1b6d47bb1d912dc1d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0e07bcd4a236450bbd303a882d5f9019, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/73f1e464146246d7b7b93d7f020b04df, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4c2e575c260a454ba2dd3cfb5003dbe1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/6dc337091d9d4cc5916568708b296e2e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/af9e9069e15d4cedbfe2e8cbbede0217, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/537a870ac2cf4dcf99056a24d270cae6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/ba908bf5eb2e45ce9d8d55b4e4340e0f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4d706510f3c84eb79b6f65b3af6f0c9a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/23324fd6ac66440394ebec49e6069a64, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/264cf405bcc442e49011a7e442859126, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5a34858c499948288b24e44fa553cb26] to archive 2024-11-16T14:16:49,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:16:49,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/196adfa8c7884ebd908987600bcc21a3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/196adfa8c7884ebd908987600bcc21a3 2024-11-16T14:16:49,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/1d7bcfc8126c4e3bad92e11612bc5b21 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/1d7bcfc8126c4e3bad92e11612bc5b21 2024-11-16T14:16:49,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/9419a1ff6df647ec87f18ffaf689c43c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/9419a1ff6df647ec87f18ffaf689c43c 2024-11-16T14:16:49,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c75124e7654e4c90a491ab3c09241476 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c75124e7654e4c90a491ab3c09241476 2024-11-16T14:16:49,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/a93cdc38fafb4572b3e656ae9878bb8c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/a93cdc38fafb4572b3e656ae9878bb8c 2024-11-16T14:16:49,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/956169663db54b80987c279c7d4b7087 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/956169663db54b80987c279c7d4b7087 2024-11-16T14:16:49,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/8aa42a0a48b944c498cb99922c1de37a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/8aa42a0a48b944c498cb99922c1de37a 2024-11-16T14:16:49,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/10c6f0d941fb40548c1ecef1b841c9f1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/10c6f0d941fb40548c1ecef1b841c9f1 2024-11-16T14:16:49,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0d665e3ddb0c4860b11e0c15b2530e92 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0d665e3ddb0c4860b11e0c15b2530e92 2024-11-16T14:16:49,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/cab720838001492bb32cfb3c176c4b2c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/cab720838001492bb32cfb3c176c4b2c 2024-11-16T14:16:49,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/08480430bb2644d186595a01c94e31f6 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/08480430bb2644d186595a01c94e31f6 2024-11-16T14:16:49,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5bd449c0d9044282b76dffbf12df5ee2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5bd449c0d9044282b76dffbf12df5ee2 2024-11-16T14:16:49,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c3339c39bcab46b5a717b38bcd544c2b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c3339c39bcab46b5a717b38bcd544c2b 2024-11-16T14:16:49,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/e6a1b547001a4a88a47af958bb3481ad to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/e6a1b547001a4a88a47af958bb3481ad 2024-11-16T14:16:49,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/b4809c8ac5a14d4281c6dc095b3ed77d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/b4809c8ac5a14d4281c6dc095b3ed77d 2024-11-16T14:16:49,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/b4c46ed48880499da696c457bcbfc09e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/b4c46ed48880499da696c457bcbfc09e 2024-11-16T14:16:49,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/6cade305c68b4ace93c863ee3e789ef9 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/6cade305c68b4ace93c863ee3e789ef9 2024-11-16T14:16:49,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4e6689c444d8494e95e632f8c9eec221 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4e6689c444d8494e95e632f8c9eec221 2024-11-16T14:16:49,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/1782bf2079ed41908409235e0900c922 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/1782bf2079ed41908409235e0900c922 2024-11-16T14:16:49,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c60410941ae44ff8bb9eef17f6642e1d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/c60410941ae44ff8bb9eef17f6642e1d 2024-11-16T14:16:49,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/ce77b5c6b9b64e7b8e0711d95d007187 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/ce77b5c6b9b64e7b8e0711d95d007187 2024-11-16T14:16:49,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5da89011ffc8438d953fcd246313678a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5da89011ffc8438d953fcd246313678a 2024-11-16T14:16:49,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/77411ebf37714f9da6b709549b14225d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/77411ebf37714f9da6b709549b14225d 2024-11-16T14:16:49,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/7c1ee7f2a14048a1b6d47bb1d912dc1d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/7c1ee7f2a14048a1b6d47bb1d912dc1d 2024-11-16T14:16:49,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0e07bcd4a236450bbd303a882d5f9019 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0e07bcd4a236450bbd303a882d5f9019 2024-11-16T14:16:49,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/73f1e464146246d7b7b93d7f020b04df to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/73f1e464146246d7b7b93d7f020b04df 2024-11-16T14:16:49,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4c2e575c260a454ba2dd3cfb5003dbe1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4c2e575c260a454ba2dd3cfb5003dbe1 2024-11-16T14:16:49,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/6dc337091d9d4cc5916568708b296e2e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/6dc337091d9d4cc5916568708b296e2e 2024-11-16T14:16:49,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/af9e9069e15d4cedbfe2e8cbbede0217 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/af9e9069e15d4cedbfe2e8cbbede0217 2024-11-16T14:16:49,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/537a870ac2cf4dcf99056a24d270cae6 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/537a870ac2cf4dcf99056a24d270cae6 2024-11-16T14:16:49,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/ba908bf5eb2e45ce9d8d55b4e4340e0f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/ba908bf5eb2e45ce9d8d55b4e4340e0f 2024-11-16T14:16:49,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4d706510f3c84eb79b6f65b3af6f0c9a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/4d706510f3c84eb79b6f65b3af6f0c9a 2024-11-16T14:16:49,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/23324fd6ac66440394ebec49e6069a64 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/23324fd6ac66440394ebec49e6069a64 2024-11-16T14:16:49,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/264cf405bcc442e49011a7e442859126 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/264cf405bcc442e49011a7e442859126 2024-11-16T14:16:49,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5a34858c499948288b24e44fa553cb26 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/5a34858c499948288b24e44fa553cb26 2024-11-16T14:16:49,826 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/recovered.edits/518.seqid, newMaxSeqId=518, maxSeqId=1 2024-11-16T14:16:49,827 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. 2024-11-16T14:16:49,827 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 4824a7508eaae82911857430ab75750c: Waiting for close lock at 1731766608190Running coprocessor pre-close hooks at 1731766608190Disabling compacts and flushes for region at 1731766608190Disabling writes for close at 1731766608487 (+297 ms)Obtaining lock to block concurrent updates at 1731766608487Preparing flush snapshotting stores in 4824a7508eaae82911857430ab75750c at 1731766608487Finished memstore snapshotting TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731766608488 (+1 ms)Flushing stores of TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c. at 1731766608488Flushing 4824a7508eaae82911857430ab75750c/A: creating writer at 1731766608488Flushing 4824a7508eaae82911857430ab75750c/A: appending metadata at 1731766608490 (+2 ms)Flushing 4824a7508eaae82911857430ab75750c/A: closing flushed file at 1731766608490Flushing 4824a7508eaae82911857430ab75750c/B: creating writer at 1731766608901 (+411 ms)Flushing 4824a7508eaae82911857430ab75750c/B: appending metadata at 1731766608903 (+2 ms)Flushing 4824a7508eaae82911857430ab75750c/B: closing flushed file at 1731766608903Flushing 4824a7508eaae82911857430ab75750c/C: creating writer at 1731766609313 (+410 ms)Flushing 4824a7508eaae82911857430ab75750c/C: appending metadata at 1731766609316 (+3 ms)Flushing 4824a7508eaae82911857430ab75750c/C: closing flushed file at 1731766609316Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22bcb4bb: reopening flushed file at 1731766609722 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aa71147: reopening flushed file at 1731766609725 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5131da8b: reopening flushed file at 1731766609728 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 4824a7508eaae82911857430ab75750c in 1244ms, sequenceid=515, compaction requested=true at 1731766609731 (+3 ms)Writing region close event to WAL at 1731766609824 (+93 ms)Running coprocessor post-close hooks at 1731766609827 (+3 ms)Closed at 1731766609827 2024-11-16T14:16:49,829 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 4824a7508eaae82911857430ab75750c 2024-11-16T14:16:49,830 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=4824a7508eaae82911857430ab75750c, regionState=CLOSED 2024-11-16T14:16:49,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:16:49,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-16T14:16:49,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 4824a7508eaae82911857430ab75750c, server=04e93f2b5fad,41317,1731766448342 in 1.7950 sec 2024-11-16T14:16:49,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-16T14:16:49,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4824a7508eaae82911857430ab75750c, UNASSIGN in 1.8000 sec 2024-11-16T14:16:49,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-16T14:16:49,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8050 sec 2024-11-16T14:16:49,839 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766609839"}]},"ts":"1731766609839"} 2024-11-16T14:16:49,841 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T14:16:49,841 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T14:16:49,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8200 sec 2024-11-16T14:16:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T14:16:50,177 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:50,177 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T14:16:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:50,180 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T14:16:50,181 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:50,185 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c 2024-11-16T14:16:50,186 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/recovered.edits] 2024-11-16T14:16:50,188 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/2d9f2bc0b50c42f4a50b512b5e676b5e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/2d9f2bc0b50c42f4a50b512b5e676b5e 2024-11-16T14:16:50,189 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/682278e5fa6c4fe0b42c32795f4d6794 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/682278e5fa6c4fe0b42c32795f4d6794 2024-11-16T14:16:50,189 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/db0f9ee1782449a285dfb171ebf34435 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/A/db0f9ee1782449a285dfb171ebf34435 2024-11-16T14:16:50,191 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9591ac8ff78e4f27b71b6a1db4e2d244 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/9591ac8ff78e4f27b71b6a1db4e2d244 2024-11-16T14:16:50,192 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c7217dbb6d6940d788e3fa8ab9c805c6 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/c7217dbb6d6940d788e3fa8ab9c805c6 2024-11-16T14:16:50,193 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/d82a2b5fed2740ed9a619ca959400612 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/B/d82a2b5fed2740ed9a619ca959400612 2024-11-16T14:16:50,194 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0351714762bf45cbb17ef1b4c8844ed3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/0351714762bf45cbb17ef1b4c8844ed3 2024-11-16T14:16:50,195 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/2e6e7feadfbe48e9ba5050abfb6d2cd9 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/2e6e7feadfbe48e9ba5050abfb6d2cd9 2024-11-16T14:16:50,197 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/f4690aa48f9a4bfebb7feda42bbe2388 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/C/f4690aa48f9a4bfebb7feda42bbe2388 2024-11-16T14:16:50,199 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/recovered.edits/518.seqid to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c/recovered.edits/518.seqid 2024-11-16T14:16:50,199 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/4824a7508eaae82911857430ab75750c 2024-11-16T14:16:50,199 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T14:16:50,202 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:50,204 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T14:16:50,206 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T14:16:50,207 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:50,207 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T14:16:50,207 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731766610207"}]},"ts":"9223372036854775807"} 2024-11-16T14:16:50,209 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T14:16:50,209 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4824a7508eaae82911857430ab75750c, NAME => 'TestAcidGuarantees,,1731766577291.4824a7508eaae82911857430ab75750c.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T14:16:50,209 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T14:16:50,209 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731766610209"}]},"ts":"9223372036854775807"} 2024-11-16T14:16:50,211 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T14:16:50,211 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:50,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-16T14:16:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T14:16:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T14:16:50,287 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:50,300 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 252), OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=381 (was 414), ProcessCount=11 (was 11), AvailableMemoryMB=4547 (was 3968) - AvailableMemoryMB LEAK? - 2024-11-16T14:16:50,311 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=381, ProcessCount=11, AvailableMemoryMB=4546 2024-11-16T14:16:50,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:16:50,312 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:16:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:50,314 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:16:50,314 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:50,315 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-16T14:16:50,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T14:16:50,315 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:16:50,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742369_1545 (size=1061) 2024-11-16T14:16:50,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T14:16:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T14:16:50,722 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc 2024-11-16T14:16:50,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742370_1546 (size=53) 2024-11-16T14:16:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T14:16:51,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:16:51,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fc05b337e1511e0b67f07003e37ed876, disabling compactions & flushes 2024-11-16T14:16:51,127 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:51,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:51,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. after waiting 0 ms 2024-11-16T14:16:51,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:51,127 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:51,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fc05b337e1511e0b67f07003e37ed876: Waiting for close lock at 1731766611127Disabling compacts and flushes for region at 1731766611127Disabling writes for close at 1731766611127Writing region close event to WAL at 1731766611127Closed at 1731766611127 2024-11-16T14:16:51,129 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:16:51,129 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731766611129"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731766611129"}]},"ts":"1731766611129"} 2024-11-16T14:16:51,131 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:16:51,132 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:16:51,133 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766611133"}]},"ts":"1731766611133"} 2024-11-16T14:16:51,134 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T14:16:51,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, ASSIGN}] 2024-11-16T14:16:51,136 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, ASSIGN 2024-11-16T14:16:51,137 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, ASSIGN; state=OFFLINE, location=04e93f2b5fad,41317,1731766448342; forceNewPlan=false, retain=false 2024-11-16T14:16:51,288 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fc05b337e1511e0b67f07003e37ed876, regionState=OPENING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:51,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, ASSIGN because future has completed 2024-11-16T14:16:51,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:16:51,451 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:51,452 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:16:51,452 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,452 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:16:51,452 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,452 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,454 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,454 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:16:51,455 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc05b337e1511e0b67f07003e37ed876 columnFamilyName A 2024-11-16T14:16:51,455 DEBUG [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:51,455 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.HStore(327): Store=fc05b337e1511e0b67f07003e37ed876/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:16:51,455 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T14:16:51,456 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:16:51,456 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc05b337e1511e0b67f07003e37ed876 columnFamilyName B 2024-11-16T14:16:51,456 DEBUG [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:51,456 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.HStore(327): Store=fc05b337e1511e0b67f07003e37ed876/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:16:51,457 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,457 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:16:51,457 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc05b337e1511e0b67f07003e37ed876 columnFamilyName C 2024-11-16T14:16:51,457 DEBUG [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:51,458 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.HStore(327): Store=fc05b337e1511e0b67f07003e37ed876/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:16:51,458 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:51,458 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,458 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,459 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,460 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,460 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,460 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:16:51,461 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,463 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:16:51,464 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened fc05b337e1511e0b67f07003e37ed876; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61139268, jitterRate=-0.0889539122581482}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:16:51,464 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:51,465 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for fc05b337e1511e0b67f07003e37ed876: Running coprocessor pre-open hook at 1731766611452Writing region info on filesystem at 1731766611452Initializing all the Stores at 1731766611453 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766611453Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766611453Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766611453Cleaning up temporary data from old regions at 1731766611460 (+7 ms)Running coprocessor post-open hooks at 1731766611464 (+4 ms)Region opened successfully at 1731766611464 2024-11-16T14:16:51,465 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., pid=148, masterSystemTime=1731766611446 2024-11-16T14:16:51,467 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:51,468 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:51,468 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fc05b337e1511e0b67f07003e37ed876, regionState=OPEN, openSeqNum=2, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:51,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:16:51,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-16T14:16:51,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 in 179 msec 2024-11-16T14:16:51,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-16T14:16:51,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, ASSIGN in 337 msec 2024-11-16T14:16:51,474 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:16:51,475 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766611475"}]},"ts":"1731766611475"} 2024-11-16T14:16:51,476 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T14:16:51,477 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:16:51,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-16T14:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T14:16:52,466 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:52,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:52,467 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:52,467 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:52,468 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d571da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:52,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:52,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:52,469 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:52,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1cb25a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:52,470 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:52,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:52,473 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:16:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 04e93f2b5fad,39231,1731766447537 2024-11-16T14:16:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a8fd43c 2024-11-16T14:16:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:16:52,474 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:16:52,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:16:52,475 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T14:16:52,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742371_1547 (size=1097) 2024-11-16T14:16:52,886 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T14:16:52,886 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T14:16:52,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:16:52,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, REOPEN/MOVE}] 2024-11-16T14:16:52,893 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, REOPEN/MOVE 2024-11-16T14:16:52,894 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=fc05b337e1511e0b67f07003e37ed876, regionState=CLOSING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:52,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, REOPEN/MOVE because future has completed 2024-11-16T14:16:52,896 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:16:52,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:16:53,050 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,050 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:16:53,050 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing fc05b337e1511e0b67f07003e37ed876, disabling compactions & flushes 2024-11-16T14:16:53,050 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,050 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,050 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. after waiting 0 ms 2024-11-16T14:16:53,050 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,058 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T14:16:53,059 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,059 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for fc05b337e1511e0b67f07003e37ed876: Waiting for close lock at 1731766613050Running coprocessor pre-close hooks at 1731766613050Disabling compacts and flushes for region at 1731766613050Disabling writes for close at 1731766613050Writing region close event to WAL at 1731766613051 (+1 ms)Running coprocessor post-close hooks at 1731766613059 (+8 ms)Closed at 1731766613059 2024-11-16T14:16:53,059 WARN [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: fc05b337e1511e0b67f07003e37ed876 to self. 2024-11-16T14:16:53,061 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,062 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=fc05b337e1511e0b67f07003e37ed876, regionState=CLOSED 2024-11-16T14:16:53,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:16:53,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-16T14:16:53,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 in 168 msec 2024-11-16T14:16:53,066 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, REOPEN/MOVE; state=CLOSED, location=04e93f2b5fad,41317,1731766448342; forceNewPlan=false, retain=true 2024-11-16T14:16:53,216 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=fc05b337e1511e0b67f07003e37ed876, regionState=OPENING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, REOPEN/MOVE because future has completed 2024-11-16T14:16:53,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:16:53,375 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,375 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:16:53,376 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,376 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:16:53,376 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,377 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,378 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,379 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:16:53,379 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc05b337e1511e0b67f07003e37ed876 columnFamilyName A 2024-11-16T14:16:53,381 DEBUG [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:53,381 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.HStore(327): Store=fc05b337e1511e0b67f07003e37ed876/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:16:53,381 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,382 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:16:53,382 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc05b337e1511e0b67f07003e37ed876 columnFamilyName B 2024-11-16T14:16:53,382 DEBUG [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:53,382 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.HStore(327): Store=fc05b337e1511e0b67f07003e37ed876/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:16:53,382 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,382 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:16:53,383 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc05b337e1511e0b67f07003e37ed876 columnFamilyName C 2024-11-16T14:16:53,383 DEBUG [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:53,383 INFO [StoreOpener-fc05b337e1511e0b67f07003e37ed876-1 {}] regionserver.HStore(327): Store=fc05b337e1511e0b67f07003e37ed876/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:16:53,383 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,383 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,383 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,384 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,385 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,385 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,385 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:16:53,386 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,386 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened fc05b337e1511e0b67f07003e37ed876; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60456682, jitterRate=-0.09912523627281189}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:16:53,386 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,387 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for fc05b337e1511e0b67f07003e37ed876: Running coprocessor pre-open hook at 1731766613377Writing region info on filesystem at 1731766613377Initializing all the Stores at 1731766613378 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766613378Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766613378Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731766613378Cleaning up temporary data from old regions at 1731766613385 (+7 ms)Running coprocessor post-open hooks at 1731766613386 (+1 ms)Region opened successfully at 1731766613387 (+1 ms) 2024-11-16T14:16:53,387 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., pid=153, masterSystemTime=1731766613369 2024-11-16T14:16:53,389 DEBUG [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,389 INFO [RS_OPEN_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,389 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=fc05b337e1511e0b67f07003e37ed876, regionState=OPEN, openSeqNum=5, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:16:53,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-16T14:16:53,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 in 173 msec 2024-11-16T14:16:53,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-16T14:16:53,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, REOPEN/MOVE in 501 msec 2024-11-16T14:16:53,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-16T14:16:53,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 506 msec 2024-11-16T14:16:53,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 921 msec 2024-11-16T14:16:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-16T14:16:53,398 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:16:53,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,398 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:53,398 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:53,399 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@449193f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:53,400 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ee1dc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:53,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:53,402 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1064b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:53,404 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c26238b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:53,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:53,406 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d133bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:53,408 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:53,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1334b1e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:53,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:53,410 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5995601f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:53,411 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531f0fbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:53,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:53,413 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a901572, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:53,414 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7738ea46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:53,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:53,415 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7388b809, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:53,417 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b22b24b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:53,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:53,418 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ad2a21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:53,419 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:53,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dbcb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:53,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:53,421 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de454f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:53,422 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ba09af4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:53,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:53,425 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe44df6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:53,426 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:53,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4709db45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,427 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 04e93f2b5fad,39231,-1 for getting cluster id 2024-11-16T14:16:53,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:16:53,428 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '22fd26c9-964e-434b-a1ee-372cf8e65a16' 2024-11-16T14:16:53,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:16:53,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "22fd26c9-964e-434b-a1ee-372cf8e65a16" 2024-11-16T14:16:53,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae0cde7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [04e93f2b5fad,39231,-1] 2024-11-16T14:16:53,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:16:53,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:16:53,429 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:16:53,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631a4ed8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:16:53,429 DEBUG [Thread-2453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:53,430 DEBUG [Thread-2455 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:53,430 DEBUG [Thread-2446 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:53,430 DEBUG [Thread-2444 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:53,430 DEBUG [Thread-2440 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:53,431 DEBUG [Thread-2459 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:53,431 DEBUG [Thread-2448 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:53,431 DEBUG [Thread-2442 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:53,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:53,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:53,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:53,432 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:16:53,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:53,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:53,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:53,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-16T14:16:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:53,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:53,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:53,434 DEBUG [Thread-2451 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:16:53,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:16:53,434 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:16:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:16:53,435 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:16:53,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:53,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:16:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:16:53,436 DEBUG [Thread-2457 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:16:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:16:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:53,437 DEBUG [Thread-2457 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=04e93f2b5fad,41317,1731766448342, seqNum=-1] 2024-11-16T14:16:53,437 DEBUG [Thread-2457 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:16:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:16:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:16:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:16:53,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:53,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:16:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:16:53,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:16:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5] 2024-11-16T14:16:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:16:53,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:16:53,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:53,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:16:53,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:53,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:16:53,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:53,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766673470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766673471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766673474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766673474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766673476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d0fab697d3814ed49e11aa93ccece049_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766613443/Put/seqid=0 2024-11-16T14:16:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:16:53,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766673585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766673586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766673586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766673586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766673586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:53,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:53,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:53,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742372_1548 (size=14594) 2024-11-16T14:16:53,645 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:53,648 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d0fab697d3814ed49e11aa93ccece049_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d0fab697d3814ed49e11aa93ccece049_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:53,649 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/bc822a0f68e942bca67209da43d1acbd, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:53,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/bc822a0f68e942bca67209da43d1acbd is 175, key is test_row_0/A:col10/1731766613443/Put/seqid=0 2024-11-16T14:16:53,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742373_1549 (size=39549) 2024-11-16T14:16:53,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:53,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:53,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:53,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:16:53,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766673795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766673795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766673795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766673795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766673796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:53,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:53,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:53,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:53,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:53,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:54,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:54,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:16:54,066 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/bc822a0f68e942bca67209da43d1acbd 2024-11-16T14:16:54,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/3f969851eb4a40f799254086daafc7b6 is 50, key is test_row_0/B:col10/1731766613443/Put/seqid=0 2024-11-16T14:16:54,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742374_1550 (size=12001) 2024-11-16T14:16:54,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766674105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766674105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766674105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766674105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766674106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:54,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:54,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:54,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:54,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/3f969851eb4a40f799254086daafc7b6 2024-11-16T14:16:54,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:54,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:54,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/9ad070041e254f48b52a90140303c635 is 50, key is test_row_0/C:col10/1731766613443/Put/seqid=0 2024-11-16T14:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:16:54,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742375_1551 (size=12001) 2024-11-16T14:16:54,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/9ad070041e254f48b52a90140303c635 2024-11-16T14:16:54,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/bc822a0f68e942bca67209da43d1acbd as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/bc822a0f68e942bca67209da43d1acbd 2024-11-16T14:16:54,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/bc822a0f68e942bca67209da43d1acbd, entries=200, sequenceid=16, filesize=38.6 K 2024-11-16T14:16:54,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/3f969851eb4a40f799254086daafc7b6 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3f969851eb4a40f799254086daafc7b6 2024-11-16T14:16:54,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3f969851eb4a40f799254086daafc7b6, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T14:16:54,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/9ad070041e254f48b52a90140303c635 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/9ad070041e254f48b52a90140303c635 2024-11-16T14:16:54,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/9ad070041e254f48b52a90140303c635, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T14:16:54,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fc05b337e1511e0b67f07003e37ed876 in 1167ms, sequenceid=16, compaction requested=false 2024-11-16T14:16:54,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:16:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:54,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-16T14:16:54,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:16:54,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:54,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:16:54,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:54,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:16:54,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:54,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111659168776a79f4e06905d7b8359b3161f_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766614620/Put/seqid=0 2024-11-16T14:16:54,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766674627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766674627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766674629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742376_1552 (size=19474) 2024-11-16T14:16:54,635 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:54,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766674630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766674632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,651 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111659168776a79f4e06905d7b8359b3161f_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111659168776a79f4e06905d7b8359b3161f_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:54,652 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/3fa1bd2d55c44c538bd9a5b0c3faed80, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:54,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/3fa1bd2d55c44c538bd9a5b0c3faed80 is 175, key is test_row_0/A:col10/1731766614620/Put/seqid=0 2024-11-16T14:16:54,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:54,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742377_1553 (size=56733) 2024-11-16T14:16:54,682 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=39, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/3fa1bd2d55c44c538bd9a5b0c3faed80 2024-11-16T14:16:54,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/16419938e1154f45a7cb57f8b7c0dad1 is 50, key is test_row_0/B:col10/1731766614620/Put/seqid=0 2024-11-16T14:16:54,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742378_1554 (size=12001) 2024-11-16T14:16:54,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/16419938e1154f45a7cb57f8b7c0dad1 2024-11-16T14:16:54,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/fa21dd05e5fe48e192cae679d00c198c is 50, key is test_row_0/C:col10/1731766614620/Put/seqid=0 2024-11-16T14:16:54,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742379_1555 (size=12001) 2024-11-16T14:16:54,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766674735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766674741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766674745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766674746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766674746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:54,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:54,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766674946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766674955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766674955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766674955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766674956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:54,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:54,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:54,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:54,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:55,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:55,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:55,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:55,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/fa21dd05e5fe48e192cae679d00c198c 2024-11-16T14:16:55,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/3fa1bd2d55c44c538bd9a5b0c3faed80 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3fa1bd2d55c44c538bd9a5b0c3faed80 2024-11-16T14:16:55,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3fa1bd2d55c44c538bd9a5b0c3faed80, entries=300, sequenceid=39, filesize=55.4 K 2024-11-16T14:16:55,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/16419938e1154f45a7cb57f8b7c0dad1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/16419938e1154f45a7cb57f8b7c0dad1 2024-11-16T14:16:55,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/16419938e1154f45a7cb57f8b7c0dad1, entries=150, sequenceid=39, filesize=11.7 K 2024-11-16T14:16:55,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/fa21dd05e5fe48e192cae679d00c198c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fa21dd05e5fe48e192cae679d00c198c 2024-11-16T14:16:55,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fa21dd05e5fe48e192cae679d00c198c, entries=150, sequenceid=39, filesize=11.7 K 2024-11-16T14:16:55,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=40.25 KB/41220 for fc05b337e1511e0b67f07003e37ed876 in 524ms, sequenceid=39, compaction requested=false 2024-11-16T14:16:55,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:16:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:55,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:16:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:16:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:16:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:16:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:55,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c5538f5763914832b66da310eab687e3_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766615257/Put/seqid=0 2024-11-16T14:16:55,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:55,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742380_1556 (size=14594) 2024-11-16T14:16:55,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766615274, max timestamp=1731766615292Num uniques 859; 2024-11-16T14:16:55,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:16:55,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766675296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766675296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766675297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766675302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766675304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,362 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:16:55,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766675405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766675416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766675416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766675416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766675416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:55,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:55,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:55,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:55,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:55,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:55,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:55,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:55,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:16:55,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766675615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766675626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766675626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766675626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766675626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,683 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:55,686 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c5538f5763914832b66da310eab687e3_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c5538f5763914832b66da310eab687e3_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:55,687 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5d874eb881da441cbc572af50660ef2a, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:55,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5d874eb881da441cbc572af50660ef2a is 175, key is test_row_0/A:col10/1731766615257/Put/seqid=0 2024-11-16T14:16:55,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742381_1557 (size=39549) 2024-11-16T14:16:55,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:55,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:55,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:55,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:55,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:55,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:55,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766675925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766675936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766675936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766675936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:55,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766675936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:56,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:56,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:56,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:56,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:56,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5d874eb881da441cbc572af50660ef2a 2024-11-16T14:16:56,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/193599a8779047b8904a473b46ba12ab is 50, key is test_row_0/B:col10/1731766615257/Put/seqid=0 2024-11-16T14:16:56,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742382_1558 (size=12001) 2024-11-16T14:16:56,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:56,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:56,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:56,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:56,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:56,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:56,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:56,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:56,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766676436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:56,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766676446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:56,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766676446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:56,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766676446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:56,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766676446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:56,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:56,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:56,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:56,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:16:56,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/193599a8779047b8904a473b46ba12ab 2024-11-16T14:16:56,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/246cae78a1ca4632b4243ef8f2ee2dcb is 50, key is test_row_0/C:col10/1731766615257/Put/seqid=0 2024-11-16T14:16:56,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742383_1559 (size=12001) 2024-11-16T14:16:56,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/246cae78a1ca4632b4243ef8f2ee2dcb 2024-11-16T14:16:56,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5d874eb881da441cbc572af50660ef2a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5d874eb881da441cbc572af50660ef2a 2024-11-16T14:16:56,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5d874eb881da441cbc572af50660ef2a, entries=200, sequenceid=49, filesize=38.6 K 2024-11-16T14:16:56,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/193599a8779047b8904a473b46ba12ab as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/193599a8779047b8904a473b46ba12ab 2024-11-16T14:16:56,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/193599a8779047b8904a473b46ba12ab, entries=150, sequenceid=49, filesize=11.7 K 2024-11-16T14:16:56,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/246cae78a1ca4632b4243ef8f2ee2dcb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/246cae78a1ca4632b4243ef8f2ee2dcb 2024-11-16T14:16:56,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/246cae78a1ca4632b4243ef8f2ee2dcb, entries=150, sequenceid=49, filesize=11.7 K 2024-11-16T14:16:56,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=129.71 KB/132820 for fc05b337e1511e0b67f07003e37ed876 in 1261ms, sequenceid=49, compaction requested=true 2024-11-16T14:16:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:16:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:56,519 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:56,519 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:56,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:56,521 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:56,521 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:56,521 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/A is initiating minor compaction (all files) 2024-11-16T14:16:56,521 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/B is initiating minor compaction (all files) 2024-11-16T14:16:56,521 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/B in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,521 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/A in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,521 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/bc822a0f68e942bca67209da43d1acbd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3fa1bd2d55c44c538bd9a5b0c3faed80, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5d874eb881da441cbc572af50660ef2a] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=132.6 K 2024-11-16T14:16:56,521 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3f969851eb4a40f799254086daafc7b6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/16419938e1154f45a7cb57f8b7c0dad1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/193599a8779047b8904a473b46ba12ab] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=35.2 K 2024-11-16T14:16:56,521 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,521 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/bc822a0f68e942bca67209da43d1acbd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3fa1bd2d55c44c538bd9a5b0c3faed80, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5d874eb881da441cbc572af50660ef2a] 2024-11-16T14:16:56,522 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f969851eb4a40f799254086daafc7b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731766613443 2024-11-16T14:16:56,522 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc822a0f68e942bca67209da43d1acbd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731766613442 2024-11-16T14:16:56,522 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 16419938e1154f45a7cb57f8b7c0dad1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731766614618 2024-11-16T14:16:56,522 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fa1bd2d55c44c538bd9a5b0c3faed80, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731766613469 2024-11-16T14:16:56,522 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 193599a8779047b8904a473b46ba12ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731766614626 2024-11-16T14:16:56,522 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d874eb881da441cbc572af50660ef2a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731766614626 2024-11-16T14:16:56,526 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:56,528 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#B#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:56,528 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/891c11fad8414d548bc0c890f3350709 is 50, key is test_row_0/B:col10/1731766615257/Put/seqid=0 2024-11-16T14:16:56,529 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116af4f9fb5f238430e87cd38c2606f61ab_fc05b337e1511e0b67f07003e37ed876 store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:56,532 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116af4f9fb5f238430e87cd38c2606f61ab_fc05b337e1511e0b67f07003e37ed876, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:56,532 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116af4f9fb5f238430e87cd38c2606f61ab_fc05b337e1511e0b67f07003e37ed876 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:56,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742384_1560 (size=12104) 2024-11-16T14:16:56,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742385_1561 (size=4469) 2024-11-16T14:16:56,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:16:56,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,646 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=129.71 KB heapSize=381.64 KB 2024-11-16T14:16:56,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:16:56,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:56,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:16:56,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:56,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:16:56,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:16:56,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116594e21dc8b744c2cbf9036ea4ff44794_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766615302/Put/seqid=0 2024-11-16T14:16:56,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742386_1562 (size=12154) 2024-11-16T14:16:56,941 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/891c11fad8414d548bc0c890f3350709 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/891c11fad8414d548bc0c890f3350709 2024-11-16T14:16:56,947 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/B of fc05b337e1511e0b67f07003e37ed876 into 891c11fad8414d548bc0c890f3350709(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:56,947 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:16:56,947 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/B, priority=13, startTime=1731766616519; duration=0sec 2024-11-16T14:16:56,947 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:56,947 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:B 2024-11-16T14:16:56,947 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:56,948 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:56,948 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/C is initiating minor compaction (all files) 2024-11-16T14:16:56,948 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/C in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:56,948 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/9ad070041e254f48b52a90140303c635, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fa21dd05e5fe48e192cae679d00c198c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/246cae78a1ca4632b4243ef8f2ee2dcb] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=35.2 K 2024-11-16T14:16:56,948 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad070041e254f48b52a90140303c635, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731766613443 2024-11-16T14:16:56,949 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting fa21dd05e5fe48e192cae679d00c198c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731766614618 2024-11-16T14:16:56,949 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#A#compaction#484 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:56,949 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 246cae78a1ca4632b4243ef8f2ee2dcb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731766614626 2024-11-16T14:16:56,950 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/738b2b0e45184c94a28733dd0e2c215e is 175, key is test_row_0/A:col10/1731766615257/Put/seqid=0 2024-11-16T14:16:56,958 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:56,958 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/ad4cfcbb9a1342c9adf52daf5bce3808 is 50, key is test_row_0/C:col10/1731766615257/Put/seqid=0 2024-11-16T14:16:56,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742388_1564 (size=12104) 2024-11-16T14:16:56,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742387_1563 (size=31058) 2024-11-16T14:16:56,976 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/ad4cfcbb9a1342c9adf52daf5bce3808 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/ad4cfcbb9a1342c9adf52daf5bce3808 2024-11-16T14:16:56,977 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/738b2b0e45184c94a28733dd0e2c215e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/738b2b0e45184c94a28733dd0e2c215e 2024-11-16T14:16:56,982 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/C of fc05b337e1511e0b67f07003e37ed876 into ad4cfcbb9a1342c9adf52daf5bce3808(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:56,982 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:16:56,982 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/C, priority=13, startTime=1731766616519; duration=0sec 2024-11-16T14:16:56,982 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:56,982 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:C 2024-11-16T14:16:56,985 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/A of fc05b337e1511e0b67f07003e37ed876 into 738b2b0e45184c94a28733dd0e2c215e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:56,985 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:16:56,985 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/A, priority=13, startTime=1731766616519; duration=0sec 2024-11-16T14:16:56,985 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:56,985 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:A 2024-11-16T14:16:57,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:57,057 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116594e21dc8b744c2cbf9036ea4ff44794_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116594e21dc8b744c2cbf9036ea4ff44794_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:57,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/b8a9a82ec95d490d82654d6e4e9018bb, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:57,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/b8a9a82ec95d490d82654d6e4e9018bb is 175, key is test_row_0/A:col10/1731766615302/Put/seqid=0 2024-11-16T14:16:57,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742389_1565 (size=30955) 2024-11-16T14:16:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:57,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:16:57,461 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/b8a9a82ec95d490d82654d6e4e9018bb 2024-11-16T14:16:57,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766677459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766677460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766677461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766677462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766677465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,467 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in fc05b337e1511e0b67f07003e37ed876 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/3d9bf3ff1d074ab58debad3f52eff892 is 50, key is test_row_0/B:col10/1731766615302/Put/seqid=0 2024-11-16T14:16:57,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742390_1566 (size=12001) 2024-11-16T14:16:57,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766677575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766677575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766677575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766677575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:16:57,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766677786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766677786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766677786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:57,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766677786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:57,874 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/3d9bf3ff1d074ab58debad3f52eff892 2024-11-16T14:16:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/1b53746595c84b3ab9e06216d2a2e50c is 50, key is test_row_0/C:col10/1731766615302/Put/seqid=0 2024-11-16T14:16:57,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742391_1567 (size=12001) 2024-11-16T14:16:58,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766678096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766678096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766678096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766678096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,282 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/1b53746595c84b3ab9e06216d2a2e50c 2024-11-16T14:16:58,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/b8a9a82ec95d490d82654d6e4e9018bb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/b8a9a82ec95d490d82654d6e4e9018bb 2024-11-16T14:16:58,287 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/b8a9a82ec95d490d82654d6e4e9018bb, entries=150, sequenceid=76, filesize=30.2 K 2024-11-16T14:16:58,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/3d9bf3ff1d074ab58debad3f52eff892 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3d9bf3ff1d074ab58debad3f52eff892 2024-11-16T14:16:58,290 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3d9bf3ff1d074ab58debad3f52eff892, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T14:16:58,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/1b53746595c84b3ab9e06216d2a2e50c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/1b53746595c84b3ab9e06216d2a2e50c 2024-11-16T14:16:58,297 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/1b53746595c84b3ab9e06216d2a2e50c, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T14:16:58,298 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~381.59 KB/390752, currentSize=46.96 KB/48090 for fc05b337e1511e0b67f07003e37ed876 in 1652ms, sequenceid=76, compaction requested=false 2024-11-16T14:16:58,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:16:58,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:58,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-16T14:16:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-16T14:16:58,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-16T14:16:58,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.8640 sec 2024-11-16T14:16:58,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 4.8710 sec 2024-11-16T14:16:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:58,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:16:58,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:16:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:16:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:16:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:16:58,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fee22eb748864303961f64a81f237bcd_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766618606/Put/seqid=0 2024-11-16T14:16:58,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742392_1568 (size=12154) 2024-11-16T14:16:58,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766678628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766678630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766678632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766678633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766678735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766678746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766678746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766678746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766678945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766678956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766678956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766678956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:59,019 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:16:59,022 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fee22eb748864303961f64a81f237bcd_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fee22eb748864303961f64a81f237bcd_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:16:59,023 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/d0862de60fcc44ee8ed09976af7caf11, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:59,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/d0862de60fcc44ee8ed09976af7caf11 is 175, key is test_row_0/A:col10/1731766618606/Put/seqid=0 2024-11-16T14:16:59,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742393_1569 (size=30955) 2024-11-16T14:16:59,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766679255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:59,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766679265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766679265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:59,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766679265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:59,433 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/d0862de60fcc44ee8ed09976af7caf11 2024-11-16T14:16:59,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/471b84c6a2fe42db9cda87d30c2f4651 is 50, key is test_row_0/B:col10/1731766618606/Put/seqid=0 2024-11-16T14:16:59,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742394_1570 (size=12001) 2024-11-16T14:16:59,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/471b84c6a2fe42db9cda87d30c2f4651 2024-11-16T14:16:59,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/fe4b770777d94712954596067e52570e is 50, key is test_row_0/C:col10/1731766618606/Put/seqid=0 2024-11-16T14:16:59,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742395_1571 (size=12001) 2024-11-16T14:16:59,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766679485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:59,487 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in fc05b337e1511e0b67f07003e37ed876 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:16:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:59,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766679765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:59,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766679775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:59,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766679776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:59,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:16:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766679776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:16:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:59,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:16:59,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:16:59,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/fe4b770777d94712954596067e52570e 2024-11-16T14:16:59,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/d0862de60fcc44ee8ed09976af7caf11 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d0862de60fcc44ee8ed09976af7caf11 2024-11-16T14:16:59,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d0862de60fcc44ee8ed09976af7caf11, entries=150, sequenceid=93, filesize=30.2 K 2024-11-16T14:16:59,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/471b84c6a2fe42db9cda87d30c2f4651 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/471b84c6a2fe42db9cda87d30c2f4651 2024-11-16T14:16:59,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/471b84c6a2fe42db9cda87d30c2f4651, entries=150, sequenceid=93, filesize=11.7 K 2024-11-16T14:16:59,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/fe4b770777d94712954596067e52570e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fe4b770777d94712954596067e52570e 2024-11-16T14:16:59,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fe4b770777d94712954596067e52570e, entries=150, sequenceid=93, filesize=11.7 K 2024-11-16T14:16:59,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fc05b337e1511e0b67f07003e37ed876 in 1257ms, sequenceid=93, compaction requested=true 2024-11-16T14:16:59,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:16:59,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:16:59,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:59,866 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:59,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:16:59,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:16:59,866 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:59,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:16:59,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:59,866 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:59,866 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:59,866 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/B is initiating minor compaction (all files) 2024-11-16T14:16:59,866 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/A is initiating minor compaction (all files) 2024-11-16T14:16:59,867 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/B in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:59,867 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/A in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:59,867 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/891c11fad8414d548bc0c890f3350709, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3d9bf3ff1d074ab58debad3f52eff892, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/471b84c6a2fe42db9cda87d30c2f4651] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=35.3 K 2024-11-16T14:16:59,867 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/738b2b0e45184c94a28733dd0e2c215e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/b8a9a82ec95d490d82654d6e4e9018bb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d0862de60fcc44ee8ed09976af7caf11] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=90.8 K 2024-11-16T14:16:59,867 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:59,867 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/738b2b0e45184c94a28733dd0e2c215e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/b8a9a82ec95d490d82654d6e4e9018bb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d0862de60fcc44ee8ed09976af7caf11] 2024-11-16T14:16:59,867 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 891c11fad8414d548bc0c890f3350709, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731766614626 2024-11-16T14:16:59,868 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 738b2b0e45184c94a28733dd0e2c215e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731766614626 2024-11-16T14:16:59,868 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d9bf3ff1d074ab58debad3f52eff892, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731766615295 2024-11-16T14:16:59,868 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8a9a82ec95d490d82654d6e4e9018bb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731766615295 2024-11-16T14:16:59,868 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0862de60fcc44ee8ed09976af7caf11, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731766618606 2024-11-16T14:16:59,868 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 471b84c6a2fe42db9cda87d30c2f4651, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731766618606 2024-11-16T14:16:59,872 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:59,873 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:59,873 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/53a2d75a84d74d74ba7a5a8f4c29b86f is 50, key is test_row_0/B:col10/1731766618606/Put/seqid=0 2024-11-16T14:16:59,874 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411168fb4374701d246e0a8d0172bd4d02cdb_fc05b337e1511e0b67f07003e37ed876 store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:59,876 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411168fb4374701d246e0a8d0172bd4d02cdb_fc05b337e1511e0b67f07003e37ed876, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:59,876 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168fb4374701d246e0a8d0172bd4d02cdb_fc05b337e1511e0b67f07003e37ed876 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:16:59,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742396_1572 (size=12207) 2024-11-16T14:16:59,885 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/53a2d75a84d74d74ba7a5a8f4c29b86f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/53a2d75a84d74d74ba7a5a8f4c29b86f 2024-11-16T14:16:59,889 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/B of fc05b337e1511e0b67f07003e37ed876 into 53a2d75a84d74d74ba7a5a8f4c29b86f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:16:59,889 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:16:59,889 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/B, priority=13, startTime=1731766619866; duration=0sec 2024-11-16T14:16:59,889 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:16:59,889 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:B 2024-11-16T14:16:59,889 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:16:59,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742397_1573 (size=4469) 2024-11-16T14:16:59,890 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:16:59,890 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/C is initiating minor compaction (all files) 2024-11-16T14:16:59,890 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/C in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:16:59,890 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/ad4cfcbb9a1342c9adf52daf5bce3808, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/1b53746595c84b3ab9e06216d2a2e50c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fe4b770777d94712954596067e52570e] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=35.3 K 2024-11-16T14:16:59,891 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#A#compaction#493 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:59,891 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ad4cfcbb9a1342c9adf52daf5bce3808, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731766614626 2024-11-16T14:16:59,891 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/93185c27bb9a4a748bad20f5d67b0212 is 175, key is test_row_0/A:col10/1731766618606/Put/seqid=0 2024-11-16T14:16:59,891 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b53746595c84b3ab9e06216d2a2e50c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731766615295 2024-11-16T14:16:59,891 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting fe4b770777d94712954596067e52570e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731766618606 2024-11-16T14:16:59,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742398_1574 (size=31161) 2024-11-16T14:16:59,896 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:16:59,897 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/ca42483839684cc9808f11f399114de9 is 50, key is test_row_0/C:col10/1731766618606/Put/seqid=0 2024-11-16T14:16:59,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742399_1575 (size=12207) 2024-11-16T14:17:00,299 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/93185c27bb9a4a748bad20f5d67b0212 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/93185c27bb9a4a748bad20f5d67b0212 2024-11-16T14:17:00,304 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/A of fc05b337e1511e0b67f07003e37ed876 into 93185c27bb9a4a748bad20f5d67b0212(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:00,304 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:00,304 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/A, priority=13, startTime=1731766619866; duration=0sec 2024-11-16T14:17:00,305 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:00,305 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:A 2024-11-16T14:17:00,307 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/ca42483839684cc9808f11f399114de9 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/ca42483839684cc9808f11f399114de9 2024-11-16T14:17:00,310 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/C of fc05b337e1511e0b67f07003e37ed876 into ca42483839684cc9808f11f399114de9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:00,310 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:00,310 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/C, priority=13, startTime=1731766619866; duration=0sec 2024-11-16T14:17:00,310 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:00,310 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:C 2024-11-16T14:17:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:00,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:17:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:00,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111669d34e0c8b3a4d1da72b573dc8b6242f_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766620777/Put/seqid=0 2024-11-16T14:17:00,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742400_1576 (size=14594) 2024-11-16T14:17:00,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766680795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:00,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766680795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:00,799 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in fc05b337e1511e0b67f07003e37ed876 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:17:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:00,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766680797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:00,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766680799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:00,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766680905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766680905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:00,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766680905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766681116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766681116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:01,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766681116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:01,188 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:01,191 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111669d34e0c8b3a4d1da72b573dc8b6242f_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111669d34e0c8b3a4d1da72b573dc8b6242f_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:01,191 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/cd863334bd1d4595b0d2f9ae057d42b8, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:01,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/cd863334bd1d4595b0d2f9ae057d42b8 is 175, key is test_row_0/A:col10/1731766620777/Put/seqid=0 2024-11-16T14:17:01,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742401_1577 (size=39549) 2024-11-16T14:17:01,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766681425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766681425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:01,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766681425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:01,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/cd863334bd1d4595b0d2f9ae057d42b8 2024-11-16T14:17:01,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/e2eaa64770d74f2ca38dacc10f128a72 is 50, key is test_row_0/B:col10/1731766620777/Put/seqid=0 2024-11-16T14:17:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:17:01,606 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:17:01,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:17:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-16T14:17:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T14:17:01,612 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:17:01,613 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:17:01,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:17:01,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742402_1578 (size=12001) 2024-11-16T14:17:01,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/e2eaa64770d74f2ca38dacc10f128a72 2024-11-16T14:17:01,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/5e366357b71b4c2c855e58a9b768b601 is 50, key is test_row_0/C:col10/1731766620777/Put/seqid=0 2024-11-16T14:17:01,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742403_1579 (size=12001) 2024-11-16T14:17:01,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/5e366357b71b4c2c855e58a9b768b601 2024-11-16T14:17:01,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/cd863334bd1d4595b0d2f9ae057d42b8 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/cd863334bd1d4595b0d2f9ae057d42b8 2024-11-16T14:17:01,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/cd863334bd1d4595b0d2f9ae057d42b8, entries=200, sequenceid=115, filesize=38.6 K 2024-11-16T14:17:01,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/e2eaa64770d74f2ca38dacc10f128a72 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/e2eaa64770d74f2ca38dacc10f128a72 2024-11-16T14:17:01,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/e2eaa64770d74f2ca38dacc10f128a72, entries=150, sequenceid=115, filesize=11.7 K 2024-11-16T14:17:01,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/5e366357b71b4c2c855e58a9b768b601 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/5e366357b71b4c2c855e58a9b768b601 2024-11-16T14:17:01,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/5e366357b71b4c2c855e58a9b768b601, entries=150, sequenceid=115, filesize=11.7 K 2024-11-16T14:17:01,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for fc05b337e1511e0b67f07003e37ed876 in 872ms, sequenceid=115, compaction requested=false 2024-11-16T14:17:01,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T14:17:01,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T14:17:01,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:01,765 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:17:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116815009f8ebd044498235573e39c9dc31_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766620798/Put/seqid=0 2024-11-16T14:17:01,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742404_1580 (size=12154) 2024-11-16T14:17:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T14:17:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:01,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:01,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766681958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:01,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766681963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766681963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:02,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766682065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:02,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766682076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:02,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:02,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766682076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:02,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:02,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:02,177 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116815009f8ebd044498235573e39c9dc31_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116815009f8ebd044498235573e39c9dc31_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:02,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/703281ac53a142a8916a13bc667962b6, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:02,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/703281ac53a142a8916a13bc667962b6 is 175, key is test_row_0/A:col10/1731766620798/Put/seqid=0 2024-11-16T14:17:02,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742405_1581 (size=30955) 2024-11-16T14:17:02,183 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/703281ac53a142a8916a13bc667962b6 2024-11-16T14:17:02,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/a8efe5ca09324f6c9db662872d201473 is 50, key is test_row_0/B:col10/1731766620798/Put/seqid=0 2024-11-16T14:17:02,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742406_1582 (size=12001) 2024-11-16T14:17:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T14:17:02,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766682275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:02,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766682286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:02,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766682286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:02,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766682585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:02,594 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/a8efe5ca09324f6c9db662872d201473 2024-11-16T14:17:02,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766682596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:02,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766682596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:02,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/f301695d8bb549e0b1bc12794978bae2 is 50, key is test_row_0/C:col10/1731766620798/Put/seqid=0 2024-11-16T14:17:02,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:02,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742407_1583 (size=12001) 2024-11-16T14:17:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T14:17:02,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766682816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:02,820 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in fc05b337e1511e0b67f07003e37ed876 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:17:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,024 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/f301695d8bb549e0b1bc12794978bae2 2024-11-16T14:17:03,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/703281ac53a142a8916a13bc667962b6 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/703281ac53a142a8916a13bc667962b6 2024-11-16T14:17:03,029 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/703281ac53a142a8916a13bc667962b6, entries=150, sequenceid=128, filesize=30.2 K 2024-11-16T14:17:03,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/a8efe5ca09324f6c9db662872d201473 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/a8efe5ca09324f6c9db662872d201473 2024-11-16T14:17:03,033 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/a8efe5ca09324f6c9db662872d201473, entries=150, sequenceid=128, filesize=11.7 K 2024-11-16T14:17:03,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/f301695d8bb549e0b1bc12794978bae2 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/f301695d8bb549e0b1bc12794978bae2 2024-11-16T14:17:03,037 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/f301695d8bb549e0b1bc12794978bae2, entries=150, sequenceid=128, filesize=11.7 K 2024-11-16T14:17:03,038 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for fc05b337e1511e0b67f07003e37ed876 in 1273ms, sequenceid=128, compaction requested=true 2024-11-16T14:17:03,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:03,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:03,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-16T14:17:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-16T14:17:03,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-16T14:17:03,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-11-16T14:17:03,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4350 sec 2024-11-16T14:17:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:03,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:17:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:03,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a6f6ccc7715a4a209c14a63c1e6ba576_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766621961/Put/seqid=0 2024-11-16T14:17:03,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766683121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742408_1584 (size=12304) 2024-11-16T14:17:03,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766683122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766683123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766683235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766683236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766683236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766683446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766683446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766683446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,528 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:03,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766683525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,530 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in fc05b337e1511e0b67f07003e37ed876 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:17:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,533 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a6f6ccc7715a4a209c14a63c1e6ba576_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a6f6ccc7715a4a209c14a63c1e6ba576_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:03,533 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/696bd9ad841c4fd3b6de318d4ba738c5, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:03,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/696bd9ad841c4fd3b6de318d4ba738c5 is 175, key is test_row_0/A:col10/1731766621961/Put/seqid=0 2024-11-16T14:17:03,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742409_1585 (size=31105) 2024-11-16T14:17:03,537 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/696bd9ad841c4fd3b6de318d4ba738c5 2024-11-16T14:17:03,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/3a12c3b0ae2c4d7cb383909789cbbc9b is 50, key is test_row_0/B:col10/1731766621961/Put/seqid=0 2024-11-16T14:17:03,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742410_1586 (size=12151) 2024-11-16T14:17:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T14:17:03,756 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:17:03,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:17:03,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-16T14:17:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766683756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766683756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:17:03,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766683756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,758 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:17:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:03,759 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:17:03,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:17:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:17:03,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:17:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:03,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:03,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/3a12c3b0ae2c4d7cb383909789cbbc9b 2024-11-16T14:17:03,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/b33ca45eeff84477b54376bc4fcccb5f is 50, key is test_row_0/C:col10/1731766621961/Put/seqid=0 2024-11-16T14:17:03,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742411_1587 (size=12151) 2024-11-16T14:17:04,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:17:04,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:04,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:04,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:04,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:04,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:04,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:17:04,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:17:04,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:04,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:04,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:04,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:04,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:04,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766684265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:04,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766684265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:04,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766684265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:04,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:17:04,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:04,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:04,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:04,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:04,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:04,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/b33ca45eeff84477b54376bc4fcccb5f 2024-11-16T14:17:04,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/696bd9ad841c4fd3b6de318d4ba738c5 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/696bd9ad841c4fd3b6de318d4ba738c5 2024-11-16T14:17:04,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/696bd9ad841c4fd3b6de318d4ba738c5, entries=150, sequenceid=148, filesize=30.4 K 2024-11-16T14:17:04,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/3a12c3b0ae2c4d7cb383909789cbbc9b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3a12c3b0ae2c4d7cb383909789cbbc9b 2024-11-16T14:17:04,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3a12c3b0ae2c4d7cb383909789cbbc9b, entries=150, sequenceid=148, filesize=11.9 K 2024-11-16T14:17:04,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/b33ca45eeff84477b54376bc4fcccb5f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/b33ca45eeff84477b54376bc4fcccb5f 2024-11-16T14:17:04,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/b33ca45eeff84477b54376bc4fcccb5f, entries=150, sequenceid=148, filesize=11.9 K 2024-11-16T14:17:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:17:04,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for fc05b337e1511e0b67f07003e37ed876 in 1289ms, sequenceid=148, compaction requested=true 2024-11-16T14:17:04,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:04,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:17:04,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:04,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:17:04,386 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:17:04,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:17:04,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:17:04,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T14:17:04,386 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:17:04,387 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132770 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:17:04,387 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/A is initiating minor compaction (all files) 2024-11-16T14:17:04,387 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/A in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:04,387 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/93185c27bb9a4a748bad20f5d67b0212, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/cd863334bd1d4595b0d2f9ae057d42b8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/703281ac53a142a8916a13bc667962b6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/696bd9ad841c4fd3b6de318d4ba738c5] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=129.7 K 2024-11-16T14:17:04,387 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:04,387 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/93185c27bb9a4a748bad20f5d67b0212, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/cd863334bd1d4595b0d2f9ae057d42b8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/703281ac53a142a8916a13bc667962b6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/696bd9ad841c4fd3b6de318d4ba738c5] 2024-11-16T14:17:04,388 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:17:04,388 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93185c27bb9a4a748bad20f5d67b0212, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731766618606 2024-11-16T14:17:04,388 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/B is initiating minor compaction (all files) 2024-11-16T14:17:04,388 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/B in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:04,388 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd863334bd1d4595b0d2f9ae057d42b8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731766618621 2024-11-16T14:17:04,388 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/53a2d75a84d74d74ba7a5a8f4c29b86f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/e2eaa64770d74f2ca38dacc10f128a72, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/a8efe5ca09324f6c9db662872d201473, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3a12c3b0ae2c4d7cb383909789cbbc9b] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=47.2 K 2024-11-16T14:17:04,388 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 703281ac53a142a8916a13bc667962b6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731766620788 2024-11-16T14:17:04,388 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 53a2d75a84d74d74ba7a5a8f4c29b86f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731766618606 2024-11-16T14:17:04,388 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 696bd9ad841c4fd3b6de318d4ba738c5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731766621961 2024-11-16T14:17:04,388 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting e2eaa64770d74f2ca38dacc10f128a72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731766618621 2024-11-16T14:17:04,388 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting a8efe5ca09324f6c9db662872d201473, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731766620788 2024-11-16T14:17:04,389 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a12c3b0ae2c4d7cb383909789cbbc9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731766621961 2024-11-16T14:17:04,394 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:04,396 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#B#compaction#505 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:04,396 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/62179c2f49074a37b99a73228a21b071 is 50, key is test_row_0/B:col10/1731766621961/Put/seqid=0 2024-11-16T14:17:04,417 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111668c5419333cb46aca8cd2581a430e6f8_fc05b337e1511e0b67f07003e37ed876 store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:04,419 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111668c5419333cb46aca8cd2581a430e6f8_fc05b337e1511e0b67f07003e37ed876, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:04,420 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111668c5419333cb46aca8cd2581a430e6f8_fc05b337e1511e0b67f07003e37ed876 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:04,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742412_1588 (size=12493) 2024-11-16T14:17:04,425 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/62179c2f49074a37b99a73228a21b071 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/62179c2f49074a37b99a73228a21b071 2024-11-16T14:17:04,428 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/B of fc05b337e1511e0b67f07003e37ed876 into 62179c2f49074a37b99a73228a21b071(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:04,428 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:04,428 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/B, priority=12, startTime=1731766624386; duration=0sec 2024-11-16T14:17:04,428 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:17:04,428 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:B 2024-11-16T14:17:04,428 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:17:04,434 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:17:04,435 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/C is initiating minor compaction (all files) 2024-11-16T14:17:04,435 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/C in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:04,435 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/ca42483839684cc9808f11f399114de9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/5e366357b71b4c2c855e58a9b768b601, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/f301695d8bb549e0b1bc12794978bae2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/b33ca45eeff84477b54376bc4fcccb5f] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=47.2 K 2024-11-16T14:17:04,435 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting ca42483839684cc9808f11f399114de9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731766618606 2024-11-16T14:17:04,435 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e366357b71b4c2c855e58a9b768b601, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731766618621 2024-11-16T14:17:04,435 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f301695d8bb549e0b1bc12794978bae2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731766620788 2024-11-16T14:17:04,436 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting b33ca45eeff84477b54376bc4fcccb5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731766621961 2024-11-16T14:17:04,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742413_1589 (size=4469) 2024-11-16T14:17:04,445 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:04,445 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/016a43c7cb9d4efaa5c3c07525929007 is 50, key is test_row_0/C:col10/1731766621961/Put/seqid=0 2024-11-16T14:17:04,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742414_1590 (size=12493) 2024-11-16T14:17:04,462 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/016a43c7cb9d4efaa5c3c07525929007 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/016a43c7cb9d4efaa5c3c07525929007 2024-11-16T14:17:04,465 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/C of fc05b337e1511e0b67f07003e37ed876 into 016a43c7cb9d4efaa5c3c07525929007(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:04,465 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:04,465 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/C, priority=12, startTime=1731766624386; duration=0sec 2024-11-16T14:17:04,465 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:04,465 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:C 2024-11-16T14:17:04,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:17:04,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:04,523 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:17:04,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:04,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:04,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:04,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:04,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:04,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:04,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c2c6177b61a24328aee8fc22430f7e37_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766623120/Put/seqid=0 2024-11-16T14:17:04,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742415_1591 (size=12304) 2024-11-16T14:17:04,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:04,538 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c2c6177b61a24328aee8fc22430f7e37_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c2c6177b61a24328aee8fc22430f7e37_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:04,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/1f96ec9b8c9542bcace38332dbb55164, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:04,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/1f96ec9b8c9542bcace38332dbb55164 is 175, key is test_row_0/A:col10/1731766623120/Put/seqid=0 2024-11-16T14:17:04,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742416_1592 (size=31105) 2024-11-16T14:17:04,834 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#A#compaction#504 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:04,834 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/152a7243faa34a88afa6d179876edec6 is 175, key is test_row_0/A:col10/1731766621961/Put/seqid=0 2024-11-16T14:17:04,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742417_1593 (size=31447) 2024-11-16T14:17:04,842 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/152a7243faa34a88afa6d179876edec6 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/152a7243faa34a88afa6d179876edec6 2024-11-16T14:17:04,846 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/A of fc05b337e1511e0b67f07003e37ed876 into 152a7243faa34a88afa6d179876edec6(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:04,846 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:04,846 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/A, priority=12, startTime=1731766624386; duration=0sec 2024-11-16T14:17:04,846 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:04,846 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:A 2024-11-16T14:17:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:17:04,950 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/1f96ec9b8c9542bcace38332dbb55164 2024-11-16T14:17:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/94f1ebfd3f474215aa6b4a8c6601c93a is 50, key is test_row_0/B:col10/1731766623120/Put/seqid=0 2024-11-16T14:17:04,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742418_1594 (size=12151) 2024-11-16T14:17:04,963 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/94f1ebfd3f474215aa6b4a8c6601c93a 2024-11-16T14:17:04,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/7123fd8c209d462fb81c2d7fd22c2211 is 50, key is test_row_0/C:col10/1731766623120/Put/seqid=0 2024-11-16T14:17:04,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742419_1595 (size=12151) 2024-11-16T14:17:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:05,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:05,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766685306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:05,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766685307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:05,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766685309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:05,379 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/7123fd8c209d462fb81c2d7fd22c2211 2024-11-16T14:17:05,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/1f96ec9b8c9542bcace38332dbb55164 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/1f96ec9b8c9542bcace38332dbb55164 2024-11-16T14:17:05,385 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/1f96ec9b8c9542bcace38332dbb55164, entries=150, sequenceid=162, filesize=30.4 K 2024-11-16T14:17:05,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/94f1ebfd3f474215aa6b4a8c6601c93a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/94f1ebfd3f474215aa6b4a8c6601c93a 2024-11-16T14:17:05,388 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/94f1ebfd3f474215aa6b4a8c6601c93a, entries=150, sequenceid=162, filesize=11.9 K 2024-11-16T14:17:05,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/7123fd8c209d462fb81c2d7fd22c2211 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/7123fd8c209d462fb81c2d7fd22c2211 2024-11-16T14:17:05,391 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/7123fd8c209d462fb81c2d7fd22c2211, entries=150, sequenceid=162, filesize=11.9 K 2024-11-16T14:17:05,392 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fc05b337e1511e0b67f07003e37ed876 in 869ms, sequenceid=162, compaction requested=false 2024-11-16T14:17:05,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:05,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:05,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-16T14:17:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-16T14:17:05,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-16T14:17:05,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6340 sec 2024-11-16T14:17:05,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.6380 sec 2024-11-16T14:17:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:05,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T14:17:05,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:05,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:17:05,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:05,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:17:05,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:05,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:17:05,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168a441515b4354edc8a542475dc0871f6_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766625416/Put/seqid=0 2024-11-16T14:17:05,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742420_1596 (size=12304) 2024-11-16T14:17:05,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766685431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766685431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:05,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766685432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:05,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766685535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:05,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766685546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:05,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766685547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:05,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766685745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:05,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766685755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:05,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766685756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:05,828 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:05,831 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168a441515b4354edc8a542475dc0871f6_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168a441515b4354edc8a542475dc0871f6_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:05,831 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5551d78385e8437680e06caa37250aac, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:05,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5551d78385e8437680e06caa37250aac is 175, key is test_row_0/A:col10/1731766625416/Put/seqid=0 2024-11-16T14:17:05,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742421_1597 (size=31105) 2024-11-16T14:17:05,835 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5551d78385e8437680e06caa37250aac 2024-11-16T14:17:05,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/9cb10da045454f1383406ab428908059 is 50, key is test_row_0/B:col10/1731766625416/Put/seqid=0 2024-11-16T14:17:05,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742422_1598 (size=12151) 2024-11-16T14:17:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:17:05,906 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:17:05,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:17:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-16T14:17:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T14:17:05,908 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:17:05,908 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:17:05,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:17:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T14:17:06,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766686056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:06,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T14:17:06,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:06,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:06,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:06,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:06,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:06,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766686065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766686065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:06,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:06,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T14:17:06,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:06,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:06,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:06,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:06,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T14:17:06,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/9cb10da045454f1383406ab428908059 2024-11-16T14:17:06,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/12583148aa5642069acf63e22d0d8c1e is 50, key is test_row_0/C:col10/1731766625416/Put/seqid=0 2024-11-16T14:17:06,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742423_1599 (size=12151) 2024-11-16T14:17:06,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/12583148aa5642069acf63e22d0d8c1e 2024-11-16T14:17:06,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5551d78385e8437680e06caa37250aac as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5551d78385e8437680e06caa37250aac 2024-11-16T14:17:06,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5551d78385e8437680e06caa37250aac, entries=150, sequenceid=184, filesize=30.4 K 2024-11-16T14:17:06,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/9cb10da045454f1383406ab428908059 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/9cb10da045454f1383406ab428908059 2024-11-16T14:17:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/9cb10da045454f1383406ab428908059, entries=150, sequenceid=184, filesize=11.9 K 2024-11-16T14:17:06,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/12583148aa5642069acf63e22d0d8c1e as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/12583148aa5642069acf63e22d0d8c1e 2024-11-16T14:17:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/12583148aa5642069acf63e22d0d8c1e, entries=150, sequenceid=184, filesize=11.9 K 2024-11-16T14:17:06,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for fc05b337e1511e0b67f07003e37ed876 in 866ms, sequenceid=184, compaction requested=true 2024-11-16T14:17:06,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:06,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:17:06,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:06,283 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:17:06,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:17:06,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:06,284 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:17:06,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:17:06,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,285 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:17:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,285 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/B is initiating minor compaction (all files) 2024-11-16T14:17:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,285 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/B in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:06,285 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/62179c2f49074a37b99a73228a21b071, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/94f1ebfd3f474215aa6b4a8c6601c93a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/9cb10da045454f1383406ab428908059] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=35.9 K 2024-11-16T14:17:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,285 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:17:06,285 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/A is initiating minor compaction (all files) 2024-11-16T14:17:06,285 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/A in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,285 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/152a7243faa34a88afa6d179876edec6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/1f96ec9b8c9542bcace38332dbb55164, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5551d78385e8437680e06caa37250aac] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=91.5 K 2024-11-16T14:17:06,285 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:06,285 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/152a7243faa34a88afa6d179876edec6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/1f96ec9b8c9542bcace38332dbb55164, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5551d78385e8437680e06caa37250aac] 2024-11-16T14:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,286 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 62179c2f49074a37b99a73228a21b071, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731766621961 2024-11-16T14:17:06,286 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 152a7243faa34a88afa6d179876edec6, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731766621961 2024-11-16T14:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,286 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 94f1ebfd3f474215aa6b4a8c6601c93a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731766623119 2024-11-16T14:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,286 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb10da045454f1383406ab428908059, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731766625299 2024-11-16T14:17:06,286 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f96ec9b8c9542bcace38332dbb55164, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731766623119 2024-11-16T14:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,287 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5551d78385e8437680e06caa37250aac, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731766625299 2024-11-16T14:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,292 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,294 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#B#compaction#514 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,294 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/7e40139d58504d02a7f30572080f0b13 is 50, key is test_row_0/B:col10/1731766625416/Put/seqid=0 2024-11-16T14:17:06,294 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116f6a345b2e25a4623a90225845cf77d4b_fc05b337e1511e0b67f07003e37ed876 store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,296 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116f6a345b2e25a4623a90225845cf77d4b_fc05b337e1511e0b67f07003e37ed876, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:06,296 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f6a345b2e25a4623a90225845cf77d4b_fc05b337e1511e0b67f07003e37ed876 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742424_1600 (size=12595) 2024-11-16T14:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,310 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/7e40139d58504d02a7f30572080f0b13 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/7e40139d58504d02a7f30572080f0b13 2024-11-16T14:17:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,316 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/B of fc05b337e1511e0b67f07003e37ed876 into 7e40139d58504d02a7f30572080f0b13(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,316 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:06,316 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/B, priority=13, startTime=1731766626284; duration=0sec 2024-11-16T14:17:06,316 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:17:06,316 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:B 2024-11-16T14:17:06,316 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,317 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,317 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/C is initiating minor compaction (all files) 2024-11-16T14:17:06,317 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/C in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:06,317 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/016a43c7cb9d4efaa5c3c07525929007, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/7123fd8c209d462fb81c2d7fd22c2211, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/12583148aa5642069acf63e22d0d8c1e] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=35.9 K 2024-11-16T14:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,317 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 016a43c7cb9d4efaa5c3c07525929007, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731766621961 2024-11-16T14:17:06,317 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7123fd8c209d462fb81c2d7fd22c2211, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731766623119 2024-11-16T14:17:06,318 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 12583148aa5642069acf63e22d0d8c1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731766625299 2024-11-16T14:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742425_1601 (size=4469) 2024-11-16T14:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,319 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#A#compaction#513 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,319 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5099441b0233464db6d3734d08856122 is 175, key is test_row_0/A:col10/1731766625416/Put/seqid=0 2024-11-16T14:17:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,323 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#C#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,324 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/091747c9d02a4a57a3b995914da31352 is 50, key is test_row_0/C:col10/1731766625416/Put/seqid=0 2024-11-16T14:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742426_1602 (size=31549) 2024-11-16T14:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742427_1603 (size=12595) 2024-11-16T14:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,343 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/091747c9d02a4a57a3b995914da31352 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/091747c9d02a4a57a3b995914da31352 2024-11-16T14:17:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,347 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/C of fc05b337e1511e0b67f07003e37ed876 into 091747c9d02a4a57a3b995914da31352(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:06,347 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:06,347 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/C, priority=13, startTime=1731766626284; duration=0sec 2024-11-16T14:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,347 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:06,347 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:C 2024-11-16T14:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T14:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:06,368 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:06,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:06,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:06,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:06,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:06,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a52b42a617a84386ba0e620756cb8a76_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766625430/Put/seqid=0 2024-11-16T14:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742428_1604 (size=9814) 2024-11-16T14:17:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,388 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a52b42a617a84386ba0e620756cb8a76_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a52b42a617a84386ba0e620756cb8a76_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/d47e6cb24177440d8c5ecc049a68fe06, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/d47e6cb24177440d8c5ecc049a68fe06 is 175, key is test_row_0/A:col10/1731766625430/Put/seqid=0 2024-11-16T14:17:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742429_1605 (size=22461) 2024-11-16T14:17:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,420 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/d47e6cb24177440d8c5ecc049a68fe06 2024-11-16T14:17:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/a5ad5808c18e482bac2d28ad63d713dc is 50, key is test_row_0/B:col10/1731766625430/Put/seqid=0 2024-11-16T14:17:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742430_1606 (size=9757) 2024-11-16T14:17:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T14:17:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:06,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766626618, max timestamp=1731766626630Num uniques 859; 2024-11-16T14:17:06,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766626618, max timestamp=1731766626630Num uniques 859; 2024-11-16T14:17:06,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:17:06,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:17:06,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731766626618, max timestamp=1731766626630Num uniques 859; 2024-11-16T14:17:06,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:17:06,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766686669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:06,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766686672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:06,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766686675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:06,721 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:17:06,731 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5099441b0233464db6d3734d08856122 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5099441b0233464db6d3734d08856122 2024-11-16T14:17:06,735 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/A of fc05b337e1511e0b67f07003e37ed876 into 5099441b0233464db6d3734d08856122(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:06,735 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:06,735 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/A, priority=13, startTime=1731766626283; duration=0sec 2024-11-16T14:17:06,735 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:06,735 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:A 2024-11-16T14:17:06,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766686786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766686786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766686786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:06,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43424 deadline: 1731766686835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,838 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in fc05b337e1511e0b67f07003e37ed876 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:17:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:06,891 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/a5ad5808c18e482bac2d28ad63d713dc 2024-11-16T14:17:06,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/3f0db82fea084ae5b227e4fc3c409dbc is 50, key is test_row_0/C:col10/1731766625430/Put/seqid=0 2024-11-16T14:17:06,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742431_1607 (size=9757) 2024-11-16T14:17:06,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766686996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:06,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766686996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766686997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T14:17:07,307 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/3f0db82fea084ae5b227e4fc3c409dbc 2024-11-16T14:17:07,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766687305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:07,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766687305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:07,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766687305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:07,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/d47e6cb24177440d8c5ecc049a68fe06 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d47e6cb24177440d8c5ecc049a68fe06 2024-11-16T14:17:07,314 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d47e6cb24177440d8c5ecc049a68fe06, entries=100, sequenceid=197, filesize=21.9 K 2024-11-16T14:17:07,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/a5ad5808c18e482bac2d28ad63d713dc as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/a5ad5808c18e482bac2d28ad63d713dc 2024-11-16T14:17:07,317 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/a5ad5808c18e482bac2d28ad63d713dc, entries=100, sequenceid=197, filesize=9.5 K 2024-11-16T14:17:07,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/3f0db82fea084ae5b227e4fc3c409dbc as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/3f0db82fea084ae5b227e4fc3c409dbc 2024-11-16T14:17:07,321 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/3f0db82fea084ae5b227e4fc3c409dbc, entries=100, sequenceid=197, filesize=9.5 K 2024-11-16T14:17:07,321 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fc05b337e1511e0b67f07003e37ed876 in 953ms, sequenceid=197, compaction requested=false 2024-11-16T14:17:07,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:07,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:07,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-16T14:17:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-16T14:17:07,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-16T14:17:07,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-11-16T14:17:07,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4200 sec 2024-11-16T14:17:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:07,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T14:17:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:17:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:17:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:17:07,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164ee53d4b0ff04d3cb04ae5b4acc68969_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766626668/Put/seqid=0 2024-11-16T14:17:07,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766687827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:07,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742432_1608 (size=12304) 2024-11-16T14:17:07,834 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:07,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:07,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766687829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:07,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766687829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:07,836 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164ee53d4b0ff04d3cb04ae5b4acc68969_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164ee53d4b0ff04d3cb04ae5b4acc68969_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:07,837 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/22e7ae67f6a64417978cc4a4d3939722, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:07,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/22e7ae67f6a64417978cc4a4d3939722 is 175, key is test_row_0/A:col10/1731766626668/Put/seqid=0 2024-11-16T14:17:07,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742433_1609 (size=31105) 2024-11-16T14:17:07,841 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/22e7ae67f6a64417978cc4a4d3939722 2024-11-16T14:17:07,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/8cb7faf2451f4b2f8e250254a20a5982 is 50, key is test_row_0/B:col10/1731766626668/Put/seqid=0 2024-11-16T14:17:07,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742434_1610 (size=12151) 2024-11-16T14:17:07,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/8cb7faf2451f4b2f8e250254a20a5982 2024-11-16T14:17:07,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/05028e63cbef43adb84515fd9c05affd is 50, key is test_row_0/C:col10/1731766626668/Put/seqid=0 2024-11-16T14:17:07,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742435_1611 (size=12151) 2024-11-16T14:17:07,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/05028e63cbef43adb84515fd9c05affd 2024-11-16T14:17:07,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/22e7ae67f6a64417978cc4a4d3939722 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/22e7ae67f6a64417978cc4a4d3939722 2024-11-16T14:17:07,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/22e7ae67f6a64417978cc4a4d3939722, entries=150, sequenceid=235, filesize=30.4 K 2024-11-16T14:17:07,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/8cb7faf2451f4b2f8e250254a20a5982 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/8cb7faf2451f4b2f8e250254a20a5982 2024-11-16T14:17:07,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/8cb7faf2451f4b2f8e250254a20a5982, entries=150, sequenceid=235, filesize=11.9 K 2024-11-16T14:17:07,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/05028e63cbef43adb84515fd9c05affd as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/05028e63cbef43adb84515fd9c05affd 2024-11-16T14:17:07,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/05028e63cbef43adb84515fd9c05affd, entries=150, sequenceid=235, filesize=11.9 K 2024-11-16T14:17:07,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for fc05b337e1511e0b67f07003e37ed876 in 70ms, sequenceid=235, compaction requested=true 2024-11-16T14:17:07,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:07,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:17:07,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:07,888 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:17:07,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:17:07,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:07,888 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:17:07,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:17:07,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:17:07,889 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:17:07,889 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/A is initiating minor compaction (all files) 2024-11-16T14:17:07,889 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/A in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:07,889 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5099441b0233464db6d3734d08856122, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d47e6cb24177440d8c5ecc049a68fe06, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/22e7ae67f6a64417978cc4a4d3939722] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=83.1 K 2024-11-16T14:17:07,889 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:07,889 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5099441b0233464db6d3734d08856122, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d47e6cb24177440d8c5ecc049a68fe06, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/22e7ae67f6a64417978cc4a4d3939722] 2024-11-16T14:17:07,890 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:17:07,890 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/B is initiating minor compaction (all files) 2024-11-16T14:17:07,890 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/B in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:07,890 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/7e40139d58504d02a7f30572080f0b13, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/a5ad5808c18e482bac2d28ad63d713dc, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/8cb7faf2451f4b2f8e250254a20a5982] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=33.7 K 2024-11-16T14:17:07,890 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5099441b0233464db6d3734d08856122, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731766625299 2024-11-16T14:17:07,890 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e40139d58504d02a7f30572080f0b13, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731766625299 2024-11-16T14:17:07,890 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d47e6cb24177440d8c5ecc049a68fe06, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731766625429 2024-11-16T14:17:07,891 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting a5ad5808c18e482bac2d28ad63d713dc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731766625429 2024-11-16T14:17:07,891 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22e7ae67f6a64417978cc4a4d3939722, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731766626663 2024-11-16T14:17:07,891 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cb7faf2451f4b2f8e250254a20a5982, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731766626663 2024-11-16T14:17:07,898 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#B#compaction#522 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:07,898 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/eefa03b7bafb4803a6a46c9fd326b40d is 50, key is test_row_0/B:col10/1731766626668/Put/seqid=0 2024-11-16T14:17:07,900 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:07,905 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116ef8bebfa111e4c1890c7c47cab85767c_fc05b337e1511e0b67f07003e37ed876 store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:07,907 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116ef8bebfa111e4c1890c7c47cab85767c_fc05b337e1511e0b67f07003e37ed876, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:07,907 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ef8bebfa111e4c1890c7c47cab85767c_fc05b337e1511e0b67f07003e37ed876 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:07,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742436_1612 (size=12697) 2024-11-16T14:17:07,926 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/eefa03b7bafb4803a6a46c9fd326b40d as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/eefa03b7bafb4803a6a46c9fd326b40d 2024-11-16T14:17:07,932 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/B of fc05b337e1511e0b67f07003e37ed876 into eefa03b7bafb4803a6a46c9fd326b40d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:07,932 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:07,932 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/B, priority=13, startTime=1731766627888; duration=0sec 2024-11-16T14:17:07,933 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:17:07,933 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:B 2024-11-16T14:17:07,933 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:17:07,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742437_1613 (size=4469) 2024-11-16T14:17:07,936 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:17:07,936 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/C is initiating minor compaction (all files) 2024-11-16T14:17:07,936 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/C in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:07,936 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/091747c9d02a4a57a3b995914da31352, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/3f0db82fea084ae5b227e4fc3c409dbc, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/05028e63cbef43adb84515fd9c05affd] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=33.7 K 2024-11-16T14:17:07,937 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#A#compaction#523 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:07,938 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/8099f4c4de1942bdaa353237fed1eaa1 is 175, key is test_row_0/A:col10/1731766626668/Put/seqid=0 2024-11-16T14:17:07,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:07,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:17:07,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:07,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:07,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:07,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:07,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:07,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:07,939 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 091747c9d02a4a57a3b995914da31352, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731766625299 2024-11-16T14:17:07,940 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f0db82fea084ae5b227e4fc3c409dbc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731766625429 2024-11-16T14:17:07,940 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 05028e63cbef43adb84515fd9c05affd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731766626663 2024-11-16T14:17:07,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742438_1614 (size=31651) 2024-11-16T14:17:07,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fa93ee75b2d74c10b80777afc4fa72dd_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766627826/Put/seqid=0 2024-11-16T14:17:07,962 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#C#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:07,963 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/82deeeab235440769e7fffc34eababb3 is 50, key is test_row_0/C:col10/1731766626668/Put/seqid=0 2024-11-16T14:17:07,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742439_1615 (size=14794) 2024-11-16T14:17:07,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742440_1616 (size=12697) 2024-11-16T14:17:07,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766627955, max timestamp=1731766627976Num uniques 859; 2024-11-16T14:17:07,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766627955, max timestamp=1731766627976Num uniques 859; 2024-11-16T14:17:07,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731766627955, max timestamp=1731766627976Num uniques 859; 2024-11-16T14:17:07,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:17:07,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:17:07,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/04e93f2b5fad:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:17:08,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766688002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766688003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766688004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T14:17:08,056 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:17:08,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:17:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-16T14:17:08,058 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:17:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T14:17:08,059 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:17:08,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:17:08,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766688116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766688116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766688116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T14:17:08,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T14:17:08,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:08,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:08,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:08,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:08,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:08,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766688326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766688326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766688326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,357 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/8099f4c4de1942bdaa353237fed1eaa1 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/8099f4c4de1942bdaa353237fed1eaa1 2024-11-16T14:17:08,362 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/A of fc05b337e1511e0b67f07003e37ed876 into 8099f4c4de1942bdaa353237fed1eaa1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:08,362 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:08,362 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/A, priority=13, startTime=1731766627888; duration=0sec 2024-11-16T14:17:08,362 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:08,362 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:A 2024-11-16T14:17:08,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T14:17:08,365 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:08,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:08,369 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fa93ee75b2d74c10b80777afc4fa72dd_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fa93ee75b2d74c10b80777afc4fa72dd_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:08,369 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/af0bc5e10e9047418bd97310dbd926fa, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:08,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/af0bc5e10e9047418bd97310dbd926fa is 175, key is test_row_0/A:col10/1731766627826/Put/seqid=0 2024-11-16T14:17:08,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742441_1617 (size=39749) 2024-11-16T14:17:08,375 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/82deeeab235440769e7fffc34eababb3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/82deeeab235440769e7fffc34eababb3 2024-11-16T14:17:08,375 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/af0bc5e10e9047418bd97310dbd926fa 2024-11-16T14:17:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T14:17:08,380 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/C of fc05b337e1511e0b67f07003e37ed876 into 82deeeab235440769e7fffc34eababb3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:08,380 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:08,380 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/C, priority=13, startTime=1731766627888; duration=0sec 2024-11-16T14:17:08,380 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:08,380 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:C 2024-11-16T14:17:08,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/513adfe3bb0a489e9bce058284c46b9c is 50, key is test_row_0/B:col10/1731766627826/Put/seqid=0 2024-11-16T14:17:08,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742442_1618 (size=12151) 2024-11-16T14:17:08,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/513adfe3bb0a489e9bce058284c46b9c 2024-11-16T14:17:08,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/aae8f69b79b44488adacdca9fec26854 is 50, key is test_row_0/C:col10/1731766627826/Put/seqid=0 2024-11-16T14:17:08,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742443_1619 (size=12151) 2024-11-16T14:17:08,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/aae8f69b79b44488adacdca9fec26854 2024-11-16T14:17:08,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/af0bc5e10e9047418bd97310dbd926fa as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/af0bc5e10e9047418bd97310dbd926fa 2024-11-16T14:17:08,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/af0bc5e10e9047418bd97310dbd926fa, entries=200, sequenceid=247, filesize=38.8 K 2024-11-16T14:17:08,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/513adfe3bb0a489e9bce058284c46b9c as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/513adfe3bb0a489e9bce058284c46b9c 2024-11-16T14:17:08,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/513adfe3bb0a489e9bce058284c46b9c, entries=150, sequenceid=247, filesize=11.9 K 2024-11-16T14:17:08,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/aae8f69b79b44488adacdca9fec26854 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/aae8f69b79b44488adacdca9fec26854 2024-11-16T14:17:08,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/aae8f69b79b44488adacdca9fec26854, entries=150, sequenceid=247, filesize=11.9 K 2024-11-16T14:17:08,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fc05b337e1511e0b67f07003e37ed876 in 480ms, sequenceid=247, compaction requested=false 2024-11-16T14:17:08,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:08,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T14:17:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:08,518 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-16T14:17:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:17:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:17:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:08,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:17:08,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169b73882563684c1e9ce4d90d78c8b7de_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766628002/Put/seqid=0 2024-11-16T14:17:08,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742444_1620 (size=12454) 2024-11-16T14:17:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:08,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:08,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766688646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766688647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766688651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T14:17:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766688755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766688755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766688766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:08,932 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169b73882563684c1e9ce4d90d78c8b7de_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169b73882563684c1e9ce4d90d78c8b7de_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:08,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/6e169a576aed4721b38e3c46b263aa78, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:08,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/6e169a576aed4721b38e3c46b263aa78 is 175, key is test_row_0/A:col10/1731766628002/Put/seqid=0 2024-11-16T14:17:08,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742445_1621 (size=31255) 2024-11-16T14:17:08,938 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/6e169a576aed4721b38e3c46b263aa78 2024-11-16T14:17:08,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/0fdb76967a774b9fa5330fb27c70c899 is 50, key is test_row_0/B:col10/1731766628002/Put/seqid=0 2024-11-16T14:17:08,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742446_1622 (size=12301) 2024-11-16T14:17:08,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766688965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766688965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:08,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766688976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T14:17:09,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766689276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:09,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766689276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:09,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766689285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:09,353 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/0fdb76967a774b9fa5330fb27c70c899 2024-11-16T14:17:09,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/8899b13179d947b8884eab6d3a603fc9 is 50, key is test_row_0/C:col10/1731766628002/Put/seqid=0 2024-11-16T14:17:09,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742447_1623 (size=12301) 2024-11-16T14:17:09,766 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/8899b13179d947b8884eab6d3a603fc9 2024-11-16T14:17:09,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/6e169a576aed4721b38e3c46b263aa78 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/6e169a576aed4721b38e3c46b263aa78 2024-11-16T14:17:09,772 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/6e169a576aed4721b38e3c46b263aa78, entries=150, sequenceid=283, filesize=30.5 K 2024-11-16T14:17:09,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/0fdb76967a774b9fa5330fb27c70c899 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/0fdb76967a774b9fa5330fb27c70c899 2024-11-16T14:17:09,775 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/0fdb76967a774b9fa5330fb27c70c899, entries=150, sequenceid=283, filesize=12.0 K 2024-11-16T14:17:09,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/8899b13179d947b8884eab6d3a603fc9 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/8899b13179d947b8884eab6d3a603fc9 2024-11-16T14:17:09,778 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/8899b13179d947b8884eab6d3a603fc9, entries=150, sequenceid=283, filesize=12.0 K 2024-11-16T14:17:09,779 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for fc05b337e1511e0b67f07003e37ed876 in 1261ms, sequenceid=283, compaction requested=true 2024-11-16T14:17:09,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:09,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:09,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-16T14:17:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-16T14:17:09,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-16T14:17:09,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7210 sec 2024-11-16T14:17:09,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:17:09,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:09,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:09,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:09,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:09,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:09,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:09,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.7260 sec 2024-11-16T14:17:09,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164a3177104cdc4358983d0d94abdb9ea9_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766628645/Put/seqid=0 2024-11-16T14:17:09,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742448_1624 (size=14994) 2024-11-16T14:17:09,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766689823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:09,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766689828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:09,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:09,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766689829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:09,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766689935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:09,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766689945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:09,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766689945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:10,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766690145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:10,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:10,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766690156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:10,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:10,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766690156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:10,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:10,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:10,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T14:17:10,205 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:17:10,205 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164a3177104cdc4358983d0d94abdb9ea9_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164a3177104cdc4358983d0d94abdb9ea9_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:10,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:17:10,206 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/3d4301734f8b41cfab07515d94b3c3d0, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-16T14:17:10,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/3d4301734f8b41cfab07515d94b3c3d0 is 175, key is test_row_0/A:col10/1731766628645/Put/seqid=0 2024-11-16T14:17:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:17:10,207 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:17:10,207 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:17:10,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:17:10,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742449_1625 (size=39949) 2024-11-16T14:17:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:17:10,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:17:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:10,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766690455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:10,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766690465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:10,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766690465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:10,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:17:10,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:10,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:10,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:10,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:17:10,610 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/3d4301734f8b41cfab07515d94b3c3d0 2024-11-16T14:17:10,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/b513a883fb6a4573a7191ca8d7c0cb43 is 50, key is test_row_0/B:col10/1731766628645/Put/seqid=0 2024-11-16T14:17:10,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742450_1626 (size=12301) 2024-11-16T14:17:10,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:17:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:10,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:17:10,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:10,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:10,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:10,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:17:10,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766690966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:10,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:17:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:10,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:10,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766690975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:10,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766690975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:11,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/b513a883fb6a4573a7191ca8d7c0cb43 2024-11-16T14:17:11,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/d262a8460dfa4c1a887cafe0fe1b5737 is 50, key is test_row_0/C:col10/1731766628645/Put/seqid=0 2024-11-16T14:17:11,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742451_1627 (size=12301) 2024-11-16T14:17:11,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:17:11,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:11,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:11,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:11,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:11,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:11,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:17:11,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:11,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:11,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:11,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:11,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:17:11,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/d262a8460dfa4c1a887cafe0fe1b5737 2024-11-16T14:17:11,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:17:11,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:11,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:11,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:11,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:11,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:17:11,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/3d4301734f8b41cfab07515d94b3c3d0 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3d4301734f8b41cfab07515d94b3c3d0 2024-11-16T14:17:11,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3d4301734f8b41cfab07515d94b3c3d0, entries=200, sequenceid=295, filesize=39.0 K 2024-11-16T14:17:11,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/b513a883fb6a4573a7191ca8d7c0cb43 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/b513a883fb6a4573a7191ca8d7c0cb43 2024-11-16T14:17:11,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/b513a883fb6a4573a7191ca8d7c0cb43, entries=150, sequenceid=295, filesize=12.0 K 2024-11-16T14:17:11,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/d262a8460dfa4c1a887cafe0fe1b5737 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/d262a8460dfa4c1a887cafe0fe1b5737 2024-11-16T14:17:11,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/d262a8460dfa4c1a887cafe0fe1b5737, entries=150, sequenceid=295, filesize=12.0 K 2024-11-16T14:17:11,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fc05b337e1511e0b67f07003e37ed876 in 1678ms, sequenceid=295, compaction requested=true 2024-11-16T14:17:11,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:11,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:17:11,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:11,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:17:11,468 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:17:11,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:17:11,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:17:11,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T14:17:11,468 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:17:11,469 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142604 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:17:11,469 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/A is initiating minor compaction (all files) 2024-11-16T14:17:11,469 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/A in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:11,469 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/8099f4c4de1942bdaa353237fed1eaa1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/af0bc5e10e9047418bd97310dbd926fa, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/6e169a576aed4721b38e3c46b263aa78, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3d4301734f8b41cfab07515d94b3c3d0] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=139.3 K 2024-11-16T14:17:11,469 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:11,470 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/8099f4c4de1942bdaa353237fed1eaa1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/af0bc5e10e9047418bd97310dbd926fa, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/6e169a576aed4721b38e3c46b263aa78, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3d4301734f8b41cfab07515d94b3c3d0] 2024-11-16T14:17:11,470 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8099f4c4de1942bdaa353237fed1eaa1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731766626663 2024-11-16T14:17:11,470 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:17:11,470 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting af0bc5e10e9047418bd97310dbd926fa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731766627823 2024-11-16T14:17:11,471 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/B is initiating minor compaction (all files) 2024-11-16T14:17:11,471 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/B in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:11,471 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/eefa03b7bafb4803a6a46c9fd326b40d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/513adfe3bb0a489e9bce058284c46b9c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/0fdb76967a774b9fa5330fb27c70c899, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/b513a883fb6a4573a7191ca8d7c0cb43] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=48.3 K 2024-11-16T14:17:11,471 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting eefa03b7bafb4803a6a46c9fd326b40d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731766626663 2024-11-16T14:17:11,471 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e169a576aed4721b38e3c46b263aa78, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731766628001 2024-11-16T14:17:11,471 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 513adfe3bb0a489e9bce058284c46b9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731766627823 2024-11-16T14:17:11,471 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d4301734f8b41cfab07515d94b3c3d0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731766628645 2024-11-16T14:17:11,472 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fdb76967a774b9fa5330fb27c70c899, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731766628001 2024-11-16T14:17:11,473 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting b513a883fb6a4573a7191ca8d7c0cb43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731766628645 2024-11-16T14:17:11,493 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:11,503 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#B#compaction#535 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:11,503 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/6dc8ad3bd7554806aca162331fa8d1e4 is 50, key is test_row_0/B:col10/1731766628645/Put/seqid=0 2024-11-16T14:17:11,509 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116795ad2b9f931451097d608f0da12104c_fc05b337e1511e0b67f07003e37ed876 store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:11,511 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116795ad2b9f931451097d608f0da12104c_fc05b337e1511e0b67f07003e37ed876, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:11,511 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116795ad2b9f931451097d608f0da12104c_fc05b337e1511e0b67f07003e37ed876 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:11,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742452_1628 (size=12983) 2024-11-16T14:17:11,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742453_1629 (size=4469) 2024-11-16T14:17:11,555 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#A#compaction#534 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:11,555 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/d8f15723c4864e0592d632c76a7df647 is 175, key is test_row_0/A:col10/1731766628645/Put/seqid=0 2024-11-16T14:17:11,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742454_1630 (size=31937) 2024-11-16T14:17:11,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:17:11,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:11,583 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:17:11,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:11,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:11,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:11,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:11,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:11,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:11,591 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/d8f15723c4864e0592d632c76a7df647 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d8f15723c4864e0592d632c76a7df647 2024-11-16T14:17:11,599 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/A of fc05b337e1511e0b67f07003e37ed876 into d8f15723c4864e0592d632c76a7df647(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:11,599 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:11,600 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/A, priority=12, startTime=1731766631468; duration=0sec 2024-11-16T14:17:11,600 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:17:11,600 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:A 2024-11-16T14:17:11,600 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:17:11,602 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:17:11,603 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/C is initiating minor compaction (all files) 2024-11-16T14:17:11,603 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/C in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:11,603 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/82deeeab235440769e7fffc34eababb3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/aae8f69b79b44488adacdca9fec26854, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/8899b13179d947b8884eab6d3a603fc9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/d262a8460dfa4c1a887cafe0fe1b5737] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=48.3 K 2024-11-16T14:17:11,603 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82deeeab235440769e7fffc34eababb3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731766626663 2024-11-16T14:17:11,604 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting aae8f69b79b44488adacdca9fec26854, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731766627823 2024-11-16T14:17:11,604 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8899b13179d947b8884eab6d3a603fc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731766628001 2024-11-16T14:17:11,605 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d262a8460dfa4c1a887cafe0fe1b5737, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731766628645 2024-11-16T14:17:11,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165cf953ab645247f7b56f1475defb6298_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766629813/Put/seqid=0 2024-11-16T14:17:11,631 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#C#compaction#537 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:11,631 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/fdba763ddd8149118ade0180c41429b3 is 50, key is test_row_0/C:col10/1731766628645/Put/seqid=0 2024-11-16T14:17:11,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742455_1631 (size=12454) 2024-11-16T14:17:11,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742456_1632 (size=12983) 2024-11-16T14:17:11,947 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/6dc8ad3bd7554806aca162331fa8d1e4 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/6dc8ad3bd7554806aca162331fa8d1e4 2024-11-16T14:17:11,950 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/B of fc05b337e1511e0b67f07003e37ed876 into 6dc8ad3bd7554806aca162331fa8d1e4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:11,950 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:11,950 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/B, priority=12, startTime=1731766631468; duration=0sec 2024-11-16T14:17:11,950 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:11,950 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:B 2024-11-16T14:17:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:11,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. as already flushing 2024-11-16T14:17:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766691997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766691998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766692000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:12,049 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165cf953ab645247f7b56f1475defb6298_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165cf953ab645247f7b56f1475defb6298_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:12,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5bfb1edb32054dfb84853e8585ee2ff5, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:12,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5bfb1edb32054dfb84853e8585ee2ff5 is 175, key is test_row_0/A:col10/1731766629813/Put/seqid=0 2024-11-16T14:17:12,054 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/fdba763ddd8149118ade0180c41429b3 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fdba763ddd8149118ade0180c41429b3 2024-11-16T14:17:12,057 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/C of fc05b337e1511e0b67f07003e37ed876 into fdba763ddd8149118ade0180c41429b3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:12,057 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:12,057 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/C, priority=12, startTime=1731766631468; duration=0sec 2024-11-16T14:17:12,057 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:12,057 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:C 2024-11-16T14:17:12,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742457_1633 (size=31255) 2024-11-16T14:17:12,059 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5bfb1edb32054dfb84853e8585ee2ff5 2024-11-16T14:17:12,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/8051927df7444972a43fca140b88111b is 50, key is test_row_0/B:col10/1731766629813/Put/seqid=0 2024-11-16T14:17:12,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742458_1634 (size=12301) 2024-11-16T14:17:12,079 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/8051927df7444972a43fca140b88111b 2024-11-16T14:17:12,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/19da0069cdae411a9e4c0f780bb6f2ba is 50, key is test_row_0/C:col10/1731766629813/Put/seqid=0 2024-11-16T14:17:12,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742459_1635 (size=12301) 2024-11-16T14:17:12,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766692105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766692105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766692105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766692315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766692316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766692316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:17:12,487 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/19da0069cdae411a9e4c0f780bb6f2ba 2024-11-16T14:17:12,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/5bfb1edb32054dfb84853e8585ee2ff5 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5bfb1edb32054dfb84853e8585ee2ff5 2024-11-16T14:17:12,496 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5bfb1edb32054dfb84853e8585ee2ff5, entries=150, sequenceid=315, filesize=30.5 K 2024-11-16T14:17:12,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/8051927df7444972a43fca140b88111b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/8051927df7444972a43fca140b88111b 2024-11-16T14:17:12,499 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/8051927df7444972a43fca140b88111b, entries=150, sequenceid=315, filesize=12.0 K 2024-11-16T14:17:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/19da0069cdae411a9e4c0f780bb6f2ba as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/19da0069cdae411a9e4c0f780bb6f2ba 2024-11-16T14:17:12,503 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/19da0069cdae411a9e4c0f780bb6f2ba, entries=150, sequenceid=315, filesize=12.0 K 2024-11-16T14:17:12,504 INFO [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for fc05b337e1511e0b67f07003e37ed876 in 920ms, sequenceid=315, compaction requested=false 2024-11-16T14:17:12,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:12,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:12,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/04e93f2b5fad:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-16T14:17:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-16T14:17:12,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-16T14:17:12,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3060 sec 2024-11-16T14:17:12,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.3090 sec 2024-11-16T14:17:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:12,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:17:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:12,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d0bc6a649e2d40a38178705f915d1781_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766632628/Put/seqid=0 2024-11-16T14:17:12,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742460_1636 (size=14994) 2024-11-16T14:17:12,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766692659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766692662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766692664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766692775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766692775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766692775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766692986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766692986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766692986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:13,050 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:13,053 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d0bc6a649e2d40a38178705f915d1781_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d0bc6a649e2d40a38178705f915d1781_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:13,054 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/7d6ab387bfd241d496ef974f05bb8231, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:13,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/7d6ab387bfd241d496ef974f05bb8231 is 175, key is test_row_0/A:col10/1731766632628/Put/seqid=0 2024-11-16T14:17:13,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742461_1637 (size=39949) 2024-11-16T14:17:13,063 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/7d6ab387bfd241d496ef974f05bb8231 2024-11-16T14:17:13,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/cb53f6322fbc4c4f8d6eb726e9e79c0a is 50, key is test_row_0/B:col10/1731766632628/Put/seqid=0 2024-11-16T14:17:13,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742462_1638 (size=12301) 2024-11-16T14:17:13,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766693295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:13,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766693296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766693296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:13,436 INFO [Thread-2457 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2457. 2024-11-16T14:17:13,436 DEBUG [Thread-2457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:17:13,436 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:13,436 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:13,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:17:13,439 INFO [Thread-2455 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2455. 2024-11-16T14:17:13,439 DEBUG [Thread-2455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:17:13,439 DEBUG [Thread-2455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:13,439 DEBUG [Thread-2455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:13,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:17:13,440 INFO [Thread-2453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2453. 2024-11-16T14:17:13,440 DEBUG [Thread-2453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:17:13,440 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:13,440 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:13,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:17:13,441 INFO [Thread-2451 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2451. 2024-11-16T14:17:13,441 DEBUG [Thread-2451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:17:13,441 DEBUG [Thread-2451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:13,441 DEBUG [Thread-2451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:13,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:17:13,444 INFO [Thread-2459 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2459. 2024-11-16T14:17:13,444 DEBUG [Thread-2459 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:17:13,444 DEBUG [Thread-2459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:13,444 DEBUG [Thread-2459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:13,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:17:13,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/cb53f6322fbc4c4f8d6eb726e9e79c0a 2024-11-16T14:17:13,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/f01b7258e25b48fb8cda446920b93b62 is 50, key is test_row_0/C:col10/1731766632628/Put/seqid=0 2024-11-16T14:17:13,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742463_1639 (size=12301) 2024-11-16T14:17:13,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731766693596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:13,597 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 04e93f2b5fad,41317,1731766448342 for 'test_row_0' in fc05b337e1511e0b67f07003e37ed876 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18294 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T14:17:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:13,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:13,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:13,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T14:17:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731766693806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1731766693806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41317 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43460 deadline: 1731766693806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 , the old value is region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:17:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., hostname=04e93f2b5fad,41317,1731766448342, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:17:13,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/f01b7258e25b48fb8cda446920b93b62 2024-11-16T14:17:13,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/7d6ab387bfd241d496ef974f05bb8231 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/7d6ab387bfd241d496ef974f05bb8231 2024-11-16T14:17:13,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/7d6ab387bfd241d496ef974f05bb8231, entries=200, sequenceid=333, filesize=39.0 K 2024-11-16T14:17:13,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/cb53f6322fbc4c4f8d6eb726e9e79c0a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/cb53f6322fbc4c4f8d6eb726e9e79c0a 2024-11-16T14:17:13,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/cb53f6322fbc4c4f8d6eb726e9e79c0a, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T14:17:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/f01b7258e25b48fb8cda446920b93b62 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/f01b7258e25b48fb8cda446920b93b62 2024-11-16T14:17:13,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/f01b7258e25b48fb8cda446920b93b62, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T14:17:13,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for fc05b337e1511e0b67f07003e37ed876 in 1282ms, sequenceid=333, compaction requested=true 2024-11-16T14:17:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:17:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:17:13,910 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:17:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc05b337e1511e0b67f07003e37ed876:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:17:13,910 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:17:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:17:13,911 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:17:13,911 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:17:13,911 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/B is initiating minor compaction (all files) 2024-11-16T14:17:13,911 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/A is initiating minor compaction (all files) 2024-11-16T14:17:13,911 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/B in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:13,911 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/A in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:13,911 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/6dc8ad3bd7554806aca162331fa8d1e4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/8051927df7444972a43fca140b88111b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/cb53f6322fbc4c4f8d6eb726e9e79c0a] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=36.7 K 2024-11-16T14:17:13,911 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d8f15723c4864e0592d632c76a7df647, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5bfb1edb32054dfb84853e8585ee2ff5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/7d6ab387bfd241d496ef974f05bb8231] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=100.7 K 2024-11-16T14:17:13,911 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:13,911 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. files: [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d8f15723c4864e0592d632c76a7df647, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5bfb1edb32054dfb84853e8585ee2ff5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/7d6ab387bfd241d496ef974f05bb8231] 2024-11-16T14:17:13,911 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc8ad3bd7554806aca162331fa8d1e4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731766628645 2024-11-16T14:17:13,912 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8f15723c4864e0592d632c76a7df647, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731766628645 2024-11-16T14:17:13,912 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 8051927df7444972a43fca140b88111b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731766629813 2024-11-16T14:17:13,912 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bfb1edb32054dfb84853e8585ee2ff5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731766629813 2024-11-16T14:17:13,912 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting cb53f6322fbc4c4f8d6eb726e9e79c0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731766631997 2024-11-16T14:17:13,912 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d6ab387bfd241d496ef974f05bb8231, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731766631997 2024-11-16T14:17:13,920 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:13,920 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#B#compaction#543 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:13,921 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/16ea84044e1b4bf5a78b88a97d66b168 is 50, key is test_row_0/B:col10/1731766632628/Put/seqid=0 2024-11-16T14:17:13,922 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116f8dc5851a1d0420db60044cf76ebacc0_fc05b337e1511e0b67f07003e37ed876 store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:13,925 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116f8dc5851a1d0420db60044cf76ebacc0_fc05b337e1511e0b67f07003e37ed876, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:13,925 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f8dc5851a1d0420db60044cf76ebacc0_fc05b337e1511e0b67f07003e37ed876 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:13,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742464_1640 (size=13085) 2024-11-16T14:17:13,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742465_1641 (size=4469) 2024-11-16T14:17:13,935 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/16ea84044e1b4bf5a78b88a97d66b168 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/16ea84044e1b4bf5a78b88a97d66b168 2024-11-16T14:17:13,939 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/B of fc05b337e1511e0b67f07003e37ed876 into 16ea84044e1b4bf5a78b88a97d66b168(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:13,939 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:13,939 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/B, priority=13, startTime=1731766633910; duration=0sec 2024-11-16T14:17:13,939 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:17:13,939 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:B 2024-11-16T14:17:13,939 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:17:13,939 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:17:13,939 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1541): fc05b337e1511e0b67f07003e37ed876/C is initiating minor compaction (all files) 2024-11-16T14:17:13,939 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc05b337e1511e0b67f07003e37ed876/C in TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:13,940 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fdba763ddd8149118ade0180c41429b3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/19da0069cdae411a9e4c0f780bb6f2ba, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/f01b7258e25b48fb8cda446920b93b62] into tmpdir=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp, totalSize=36.7 K 2024-11-16T14:17:13,940 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting fdba763ddd8149118ade0180c41429b3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731766628645 2024-11-16T14:17:13,940 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting 19da0069cdae411a9e4c0f780bb6f2ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731766629813 2024-11-16T14:17:13,940 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] compactions.Compactor(225): Compacting f01b7258e25b48fb8cda446920b93b62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731766631997 2024-11-16T14:17:13,946 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#C#compaction#545 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:13,946 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/9bbab071b57d4b61ab0023352e07176b is 50, key is test_row_0/C:col10/1731766632628/Put/seqid=0 2024-11-16T14:17:13,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742466_1642 (size=13085) 2024-11-16T14:17:14,335 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc05b337e1511e0b67f07003e37ed876#A#compaction#544 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:17:14,335 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/3aef8617e6ee4421b2ec998aecbbbbe6 is 175, key is test_row_0/A:col10/1731766632628/Put/seqid=0 2024-11-16T14:17:14,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742467_1643 (size=32039) 2024-11-16T14:17:14,352 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/9bbab071b57d4b61ab0023352e07176b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/9bbab071b57d4b61ab0023352e07176b 2024-11-16T14:17:14,355 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/C of fc05b337e1511e0b67f07003e37ed876 into 9bbab071b57d4b61ab0023352e07176b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:14,355 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:14,355 INFO [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/C, priority=13, startTime=1731766633910; duration=0sec 2024-11-16T14:17:14,355 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:14,355 DEBUG [RS:0;04e93f2b5fad:41317-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:C 2024-11-16T14:17:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:17:14,386 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:17:14,742 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/3aef8617e6ee4421b2ec998aecbbbbe6 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3aef8617e6ee4421b2ec998aecbbbbe6 2024-11-16T14:17:14,745 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc05b337e1511e0b67f07003e37ed876/A of fc05b337e1511e0b67f07003e37ed876 into 3aef8617e6ee4421b2ec998aecbbbbe6(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:17:14,745 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:14,745 INFO [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., storeName=fc05b337e1511e0b67f07003e37ed876/A, priority=13, startTime=1731766633910; duration=0sec 2024-11-16T14:17:14,745 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:17:14,745 DEBUG [RS:0;04e93f2b5fad:41317-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc05b337e1511e0b67f07003e37ed876:A 2024-11-16T14:17:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41317 {}] regionserver.HRegion(8855): Flush requested on fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:14,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:17:14,818 INFO [Thread-2448 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2448. 2024-11-16T14:17:14,818 INFO [Thread-2442 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2442. 2024-11-16T14:17:14,818 DEBUG [Thread-2442 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:17:14,818 DEBUG [Thread-2448 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:17:14,818 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:14,818 DEBUG [Thread-2448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:14,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:14,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:14,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:14,818 DEBUG [Thread-2448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:14,818 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:14,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:14,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:14,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:17:14,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:14,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:17:14,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111600b9fe7c0fa3477ab2562ad69475c444_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766634816/Put/seqid=0 2024-11-16T14:17:14,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742468_1644 (size=12454) 2024-11-16T14:17:14,826 INFO [Thread-2446 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2446. 2024-11-16T14:17:14,826 DEBUG [Thread-2446 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:17:14,826 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:14,826 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:14,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:17:15,225 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:15,228 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111600b9fe7c0fa3477ab2562ad69475c444_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111600b9fe7c0fa3477ab2562ad69475c444_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:15,229 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/9b3ca5a1ab5c4cb9be4a1cd6220db660, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:15,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/9b3ca5a1ab5c4cb9be4a1cd6220db660 is 175, key is test_row_0/A:col10/1731766634816/Put/seqid=0 2024-11-16T14:17:15,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742469_1645 (size=31255) 2024-11-16T14:17:15,632 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/9b3ca5a1ab5c4cb9be4a1cd6220db660 2024-11-16T14:17:15,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/96830afd3def4010915f7702d5fce1b8 is 50, key is test_row_0/B:col10/1731766634816/Put/seqid=0 2024-11-16T14:17:15,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742470_1646 (size=12301) 2024-11-16T14:17:16,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/96830afd3def4010915f7702d5fce1b8 2024-11-16T14:17:16,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/0842fc5410654a2696b09923deb8663f is 50, key is test_row_0/C:col10/1731766634816/Put/seqid=0 2024-11-16T14:17:16,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742471_1647 (size=12301) 2024-11-16T14:17:16,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/0842fc5410654a2696b09923deb8663f 2024-11-16T14:17:16,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/9b3ca5a1ab5c4cb9be4a1cd6220db660 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/9b3ca5a1ab5c4cb9be4a1cd6220db660 2024-11-16T14:17:16,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/9b3ca5a1ab5c4cb9be4a1cd6220db660, entries=150, sequenceid=355, filesize=30.5 K 2024-11-16T14:17:16,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/96830afd3def4010915f7702d5fce1b8 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/96830afd3def4010915f7702d5fce1b8 2024-11-16T14:17:16,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/96830afd3def4010915f7702d5fce1b8, entries=150, sequenceid=355, filesize=12.0 K 2024-11-16T14:17:16,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/0842fc5410654a2696b09923deb8663f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/0842fc5410654a2696b09923deb8663f 2024-11-16T14:17:16,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/0842fc5410654a2696b09923deb8663f, entries=150, sequenceid=355, filesize=12.0 K 2024-11-16T14:17:16,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for fc05b337e1511e0b67f07003e37ed876 in 1642ms, sequenceid=355, compaction requested=false 2024-11-16T14:17:16,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc05b337e1511e0b67f07003e37ed876: 2024-11-16T14:17:16,896 INFO [Thread-2444 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2444. 2024-11-16T14:17:16,896 DEBUG [Thread-2444 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:17:16,896 DEBUG [Thread-2444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:16,897 DEBUG [Thread-2444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:16,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:17:23,667 INFO [Thread-2440 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2440. 2024-11-16T14:17:23,667 DEBUG [Thread-2440 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:17:23,667 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:23,667 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:23,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2986 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3199 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2995 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2979 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3197 2024-11-16T14:17:23,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T14:17:23,667 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:17:23,668 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:17:23,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:23,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:23,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:17:23,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T14:17:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T14:17:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T14:17:23,671 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766643671"}]},"ts":"1731766643671"} 2024-11-16T14:17:23,672 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T14:17:23,672 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T14:17:23,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:17:23,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, UNASSIGN}] 2024-11-16T14:17:23,674 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, UNASSIGN 2024-11-16T14:17:23,675 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=fc05b337e1511e0b67f07003e37ed876, regionState=CLOSING, regionLocation=04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:23,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, UNASSIGN because future has completed 2024-11-16T14:17:23,676 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:17:23,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342}] 2024-11-16T14:17:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T14:17:23,829 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:23,829 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:17:23,829 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing fc05b337e1511e0b67f07003e37ed876, disabling compactions & flushes 2024-11-16T14:17:23,829 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:23,829 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:23,829 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. after waiting 0 ms 2024-11-16T14:17:23,829 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:23,829 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing fc05b337e1511e0b67f07003e37ed876 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-16T14:17:23,829 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=A 2024-11-16T14:17:23,829 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:23,829 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=B 2024-11-16T14:17:23,829 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:23,829 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fc05b337e1511e0b67f07003e37ed876, store=C 2024-11-16T14:17:23,829 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:17:23,833 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168062aad20b914b44b768f27445a86b97_fc05b337e1511e0b67f07003e37ed876 is 50, key is test_row_0/A:col10/1731766643666/Put/seqid=0 2024-11-16T14:17:23,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742472_1648 (size=9914) 2024-11-16T14:17:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T14:17:24,236 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:17:24,240 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168062aad20b914b44b768f27445a86b97_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168062aad20b914b44b768f27445a86b97_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:24,241 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/2fdc767e9fa0467b8c7d0057672f135f, store: [table=TestAcidGuarantees family=A region=fc05b337e1511e0b67f07003e37ed876] 2024-11-16T14:17:24,241 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/2fdc767e9fa0467b8c7d0057672f135f is 175, key is test_row_0/A:col10/1731766643666/Put/seqid=0 2024-11-16T14:17:24,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742473_1649 (size=22561) 2024-11-16T14:17:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T14:17:24,644 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/2fdc767e9fa0467b8c7d0057672f135f 2024-11-16T14:17:24,649 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/496a3b83bc544b338b84021586f9a279 is 50, key is test_row_0/B:col10/1731766643666/Put/seqid=0 2024-11-16T14:17:24,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742474_1650 (size=9857) 2024-11-16T14:17:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T14:17:25,052 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/496a3b83bc544b338b84021586f9a279 2024-11-16T14:17:25,057 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/67edd4331ebf4b60b5974f52445f644a is 50, key is test_row_0/C:col10/1731766643666/Put/seqid=0 2024-11-16T14:17:25,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742475_1651 (size=9857) 2024-11-16T14:17:25,460 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/67edd4331ebf4b60b5974f52445f644a 2024-11-16T14:17:25,464 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/A/2fdc767e9fa0467b8c7d0057672f135f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/2fdc767e9fa0467b8c7d0057672f135f 2024-11-16T14:17:25,466 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/2fdc767e9fa0467b8c7d0057672f135f, entries=100, sequenceid=361, filesize=22.0 K 2024-11-16T14:17:25,467 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/B/496a3b83bc544b338b84021586f9a279 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/496a3b83bc544b338b84021586f9a279 2024-11-16T14:17:25,469 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/496a3b83bc544b338b84021586f9a279, entries=100, sequenceid=361, filesize=9.6 K 2024-11-16T14:17:25,469 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/.tmp/C/67edd4331ebf4b60b5974f52445f644a as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/67edd4331ebf4b60b5974f52445f644a 2024-11-16T14:17:25,471 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/67edd4331ebf4b60b5974f52445f644a, entries=100, sequenceid=361, filesize=9.6 K 2024-11-16T14:17:25,472 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fc05b337e1511e0b67f07003e37ed876 in 1643ms, sequenceid=361, compaction requested=true 2024-11-16T14:17:25,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/bc822a0f68e942bca67209da43d1acbd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3fa1bd2d55c44c538bd9a5b0c3faed80, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5d874eb881da441cbc572af50660ef2a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/738b2b0e45184c94a28733dd0e2c215e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/b8a9a82ec95d490d82654d6e4e9018bb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/93185c27bb9a4a748bad20f5d67b0212, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d0862de60fcc44ee8ed09976af7caf11, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/cd863334bd1d4595b0d2f9ae057d42b8, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/703281ac53a142a8916a13bc667962b6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/152a7243faa34a88afa6d179876edec6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/696bd9ad841c4fd3b6de318d4ba738c5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/1f96ec9b8c9542bcace38332dbb55164, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5099441b0233464db6d3734d08856122, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5551d78385e8437680e06caa37250aac, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d47e6cb24177440d8c5ecc049a68fe06, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/8099f4c4de1942bdaa353237fed1eaa1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/22e7ae67f6a64417978cc4a4d3939722, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/af0bc5e10e9047418bd97310dbd926fa, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/6e169a576aed4721b38e3c46b263aa78, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3d4301734f8b41cfab07515d94b3c3d0, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d8f15723c4864e0592d632c76a7df647, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5bfb1edb32054dfb84853e8585ee2ff5, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/7d6ab387bfd241d496ef974f05bb8231] to archive 2024-11-16T14:17:25,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:17:25,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/bc822a0f68e942bca67209da43d1acbd to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/bc822a0f68e942bca67209da43d1acbd 2024-11-16T14:17:25,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3fa1bd2d55c44c538bd9a5b0c3faed80 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3fa1bd2d55c44c538bd9a5b0c3faed80 2024-11-16T14:17:25,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5d874eb881da441cbc572af50660ef2a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5d874eb881da441cbc572af50660ef2a 2024-11-16T14:17:25,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/738b2b0e45184c94a28733dd0e2c215e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/738b2b0e45184c94a28733dd0e2c215e 2024-11-16T14:17:25,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/b8a9a82ec95d490d82654d6e4e9018bb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/b8a9a82ec95d490d82654d6e4e9018bb 2024-11-16T14:17:25,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/93185c27bb9a4a748bad20f5d67b0212 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/93185c27bb9a4a748bad20f5d67b0212 2024-11-16T14:17:25,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d0862de60fcc44ee8ed09976af7caf11 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d0862de60fcc44ee8ed09976af7caf11 2024-11-16T14:17:25,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/cd863334bd1d4595b0d2f9ae057d42b8 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/cd863334bd1d4595b0d2f9ae057d42b8 2024-11-16T14:17:25,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/703281ac53a142a8916a13bc667962b6 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/703281ac53a142a8916a13bc667962b6 2024-11-16T14:17:25,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/152a7243faa34a88afa6d179876edec6 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/152a7243faa34a88afa6d179876edec6 2024-11-16T14:17:25,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/696bd9ad841c4fd3b6de318d4ba738c5 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/696bd9ad841c4fd3b6de318d4ba738c5 2024-11-16T14:17:25,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/1f96ec9b8c9542bcace38332dbb55164 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/1f96ec9b8c9542bcace38332dbb55164 2024-11-16T14:17:25,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5099441b0233464db6d3734d08856122 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5099441b0233464db6d3734d08856122 2024-11-16T14:17:25,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5551d78385e8437680e06caa37250aac to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5551d78385e8437680e06caa37250aac 2024-11-16T14:17:25,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d47e6cb24177440d8c5ecc049a68fe06 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d47e6cb24177440d8c5ecc049a68fe06 2024-11-16T14:17:25,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/8099f4c4de1942bdaa353237fed1eaa1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/8099f4c4de1942bdaa353237fed1eaa1 2024-11-16T14:17:25,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/22e7ae67f6a64417978cc4a4d3939722 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/22e7ae67f6a64417978cc4a4d3939722 2024-11-16T14:17:25,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/af0bc5e10e9047418bd97310dbd926fa to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/af0bc5e10e9047418bd97310dbd926fa 2024-11-16T14:17:25,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/6e169a576aed4721b38e3c46b263aa78 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/6e169a576aed4721b38e3c46b263aa78 2024-11-16T14:17:25,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3d4301734f8b41cfab07515d94b3c3d0 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3d4301734f8b41cfab07515d94b3c3d0 2024-11-16T14:17:25,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d8f15723c4864e0592d632c76a7df647 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/d8f15723c4864e0592d632c76a7df647 2024-11-16T14:17:25,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5bfb1edb32054dfb84853e8585ee2ff5 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/5bfb1edb32054dfb84853e8585ee2ff5 2024-11-16T14:17:25,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/7d6ab387bfd241d496ef974f05bb8231 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/7d6ab387bfd241d496ef974f05bb8231 2024-11-16T14:17:25,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3f969851eb4a40f799254086daafc7b6, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/16419938e1154f45a7cb57f8b7c0dad1, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/891c11fad8414d548bc0c890f3350709, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/193599a8779047b8904a473b46ba12ab, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3d9bf3ff1d074ab58debad3f52eff892, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/53a2d75a84d74d74ba7a5a8f4c29b86f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/471b84c6a2fe42db9cda87d30c2f4651, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/e2eaa64770d74f2ca38dacc10f128a72, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/a8efe5ca09324f6c9db662872d201473, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/62179c2f49074a37b99a73228a21b071, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3a12c3b0ae2c4d7cb383909789cbbc9b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/94f1ebfd3f474215aa6b4a8c6601c93a, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/7e40139d58504d02a7f30572080f0b13, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/9cb10da045454f1383406ab428908059, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/a5ad5808c18e482bac2d28ad63d713dc, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/eefa03b7bafb4803a6a46c9fd326b40d, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/8cb7faf2451f4b2f8e250254a20a5982, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/513adfe3bb0a489e9bce058284c46b9c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/0fdb76967a774b9fa5330fb27c70c899, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/6dc8ad3bd7554806aca162331fa8d1e4, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/b513a883fb6a4573a7191ca8d7c0cb43, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/8051927df7444972a43fca140b88111b, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/cb53f6322fbc4c4f8d6eb726e9e79c0a] to archive 2024-11-16T14:17:25,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:17:25,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3f969851eb4a40f799254086daafc7b6 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3f969851eb4a40f799254086daafc7b6 2024-11-16T14:17:25,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/16419938e1154f45a7cb57f8b7c0dad1 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/16419938e1154f45a7cb57f8b7c0dad1 2024-11-16T14:17:25,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/891c11fad8414d548bc0c890f3350709 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/891c11fad8414d548bc0c890f3350709 2024-11-16T14:17:25,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/193599a8779047b8904a473b46ba12ab to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/193599a8779047b8904a473b46ba12ab 2024-11-16T14:17:25,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3d9bf3ff1d074ab58debad3f52eff892 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3d9bf3ff1d074ab58debad3f52eff892 2024-11-16T14:17:25,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/53a2d75a84d74d74ba7a5a8f4c29b86f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/53a2d75a84d74d74ba7a5a8f4c29b86f 2024-11-16T14:17:25,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/471b84c6a2fe42db9cda87d30c2f4651 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/471b84c6a2fe42db9cda87d30c2f4651 2024-11-16T14:17:25,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/e2eaa64770d74f2ca38dacc10f128a72 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/e2eaa64770d74f2ca38dacc10f128a72 2024-11-16T14:17:25,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/a8efe5ca09324f6c9db662872d201473 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/a8efe5ca09324f6c9db662872d201473 2024-11-16T14:17:25,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/62179c2f49074a37b99a73228a21b071 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/62179c2f49074a37b99a73228a21b071 2024-11-16T14:17:25,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3a12c3b0ae2c4d7cb383909789cbbc9b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/3a12c3b0ae2c4d7cb383909789cbbc9b 2024-11-16T14:17:25,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/94f1ebfd3f474215aa6b4a8c6601c93a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/94f1ebfd3f474215aa6b4a8c6601c93a 2024-11-16T14:17:25,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/7e40139d58504d02a7f30572080f0b13 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/7e40139d58504d02a7f30572080f0b13 2024-11-16T14:17:25,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/9cb10da045454f1383406ab428908059 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/9cb10da045454f1383406ab428908059 2024-11-16T14:17:25,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/a5ad5808c18e482bac2d28ad63d713dc to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/a5ad5808c18e482bac2d28ad63d713dc 2024-11-16T14:17:25,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/eefa03b7bafb4803a6a46c9fd326b40d to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/eefa03b7bafb4803a6a46c9fd326b40d 2024-11-16T14:17:25,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/8cb7faf2451f4b2f8e250254a20a5982 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/8cb7faf2451f4b2f8e250254a20a5982 2024-11-16T14:17:25,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/513adfe3bb0a489e9bce058284c46b9c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/513adfe3bb0a489e9bce058284c46b9c 2024-11-16T14:17:25,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/0fdb76967a774b9fa5330fb27c70c899 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/0fdb76967a774b9fa5330fb27c70c899 2024-11-16T14:17:25,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/6dc8ad3bd7554806aca162331fa8d1e4 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/6dc8ad3bd7554806aca162331fa8d1e4 2024-11-16T14:17:25,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/b513a883fb6a4573a7191ca8d7c0cb43 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/b513a883fb6a4573a7191ca8d7c0cb43 2024-11-16T14:17:25,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/8051927df7444972a43fca140b88111b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/8051927df7444972a43fca140b88111b 2024-11-16T14:17:25,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/cb53f6322fbc4c4f8d6eb726e9e79c0a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/cb53f6322fbc4c4f8d6eb726e9e79c0a 2024-11-16T14:17:25,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/9ad070041e254f48b52a90140303c635, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fa21dd05e5fe48e192cae679d00c198c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/ad4cfcbb9a1342c9adf52daf5bce3808, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/246cae78a1ca4632b4243ef8f2ee2dcb, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/1b53746595c84b3ab9e06216d2a2e50c, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/ca42483839684cc9808f11f399114de9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fe4b770777d94712954596067e52570e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/5e366357b71b4c2c855e58a9b768b601, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/f301695d8bb549e0b1bc12794978bae2, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/016a43c7cb9d4efaa5c3c07525929007, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/b33ca45eeff84477b54376bc4fcccb5f, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/7123fd8c209d462fb81c2d7fd22c2211, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/091747c9d02a4a57a3b995914da31352, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/12583148aa5642069acf63e22d0d8c1e, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/3f0db82fea084ae5b227e4fc3c409dbc, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/82deeeab235440769e7fffc34eababb3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/05028e63cbef43adb84515fd9c05affd, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/aae8f69b79b44488adacdca9fec26854, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/8899b13179d947b8884eab6d3a603fc9, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fdba763ddd8149118ade0180c41429b3, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/d262a8460dfa4c1a887cafe0fe1b5737, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/19da0069cdae411a9e4c0f780bb6f2ba, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/f01b7258e25b48fb8cda446920b93b62] to archive 2024-11-16T14:17:25,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:17:25,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/9ad070041e254f48b52a90140303c635 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/9ad070041e254f48b52a90140303c635 2024-11-16T14:17:25,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fa21dd05e5fe48e192cae679d00c198c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fa21dd05e5fe48e192cae679d00c198c 2024-11-16T14:17:25,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/ad4cfcbb9a1342c9adf52daf5bce3808 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/ad4cfcbb9a1342c9adf52daf5bce3808 2024-11-16T14:17:25,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/246cae78a1ca4632b4243ef8f2ee2dcb to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/246cae78a1ca4632b4243ef8f2ee2dcb 2024-11-16T14:17:25,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/1b53746595c84b3ab9e06216d2a2e50c to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/1b53746595c84b3ab9e06216d2a2e50c 2024-11-16T14:17:25,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/ca42483839684cc9808f11f399114de9 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/ca42483839684cc9808f11f399114de9 2024-11-16T14:17:25,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fe4b770777d94712954596067e52570e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fe4b770777d94712954596067e52570e 2024-11-16T14:17:25,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/5e366357b71b4c2c855e58a9b768b601 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/5e366357b71b4c2c855e58a9b768b601 2024-11-16T14:17:25,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/f301695d8bb549e0b1bc12794978bae2 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/f301695d8bb549e0b1bc12794978bae2 2024-11-16T14:17:25,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/016a43c7cb9d4efaa5c3c07525929007 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/016a43c7cb9d4efaa5c3c07525929007 2024-11-16T14:17:25,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/b33ca45eeff84477b54376bc4fcccb5f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/b33ca45eeff84477b54376bc4fcccb5f 2024-11-16T14:17:25,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/7123fd8c209d462fb81c2d7fd22c2211 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/7123fd8c209d462fb81c2d7fd22c2211 2024-11-16T14:17:25,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/091747c9d02a4a57a3b995914da31352 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/091747c9d02a4a57a3b995914da31352 2024-11-16T14:17:25,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/12583148aa5642069acf63e22d0d8c1e to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/12583148aa5642069acf63e22d0d8c1e 2024-11-16T14:17:25,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/3f0db82fea084ae5b227e4fc3c409dbc to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/3f0db82fea084ae5b227e4fc3c409dbc 2024-11-16T14:17:25,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/82deeeab235440769e7fffc34eababb3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/82deeeab235440769e7fffc34eababb3 2024-11-16T14:17:25,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/05028e63cbef43adb84515fd9c05affd to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/05028e63cbef43adb84515fd9c05affd 2024-11-16T14:17:25,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/aae8f69b79b44488adacdca9fec26854 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/aae8f69b79b44488adacdca9fec26854 2024-11-16T14:17:25,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/8899b13179d947b8884eab6d3a603fc9 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/8899b13179d947b8884eab6d3a603fc9 2024-11-16T14:17:25,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fdba763ddd8149118ade0180c41429b3 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/fdba763ddd8149118ade0180c41429b3 2024-11-16T14:17:25,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/d262a8460dfa4c1a887cafe0fe1b5737 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/d262a8460dfa4c1a887cafe0fe1b5737 2024-11-16T14:17:25,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/19da0069cdae411a9e4c0f780bb6f2ba to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/19da0069cdae411a9e4c0f780bb6f2ba 2024-11-16T14:17:25,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/f01b7258e25b48fb8cda446920b93b62 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/f01b7258e25b48fb8cda446920b93b62 2024-11-16T14:17:25,523 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/recovered.edits/364.seqid, newMaxSeqId=364, maxSeqId=4 2024-11-16T14:17:25,524 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. 2024-11-16T14:17:25,524 DEBUG [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for fc05b337e1511e0b67f07003e37ed876: Waiting for close lock at 1731766643829Running coprocessor pre-close hooks at 1731766643829Disabling compacts and flushes for region at 1731766643829Disabling writes for close at 1731766643829Obtaining lock to block concurrent updates at 1731766643829Preparing flush snapshotting stores in fc05b337e1511e0b67f07003e37ed876 at 1731766643829Finished memstore snapshotting TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731766643829Flushing stores of TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876. at 1731766643830 (+1 ms)Flushing fc05b337e1511e0b67f07003e37ed876/A: creating writer at 1731766643830Flushing mob file fc05b337e1511e0b67f07003e37ed876/A: appending metadata at 1731766643833 (+3 ms)Flushing mob file fc05b337e1511e0b67f07003e37ed876/A: closing flushed file at 1731766643833Flushing fc05b337e1511e0b67f07003e37ed876/A: appending metadata at 1731766644241 (+408 ms)Flushing fc05b337e1511e0b67f07003e37ed876/A: closing flushed file at 1731766644241Flushing fc05b337e1511e0b67f07003e37ed876/B: creating writer at 1731766644647 (+406 ms)Flushing fc05b337e1511e0b67f07003e37ed876/B: appending metadata at 1731766644649 (+2 ms)Flushing fc05b337e1511e0b67f07003e37ed876/B: closing flushed file at 1731766644649Flushing fc05b337e1511e0b67f07003e37ed876/C: creating writer at 1731766645055 (+406 ms)Flushing fc05b337e1511e0b67f07003e37ed876/C: appending metadata at 1731766645057 (+2 ms)Flushing fc05b337e1511e0b67f07003e37ed876/C: closing flushed file at 1731766645057Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e41da72: reopening flushed file at 1731766645463 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a94ce09: reopening flushed file at 1731766645466 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e88fa95: reopening flushed file at 1731766645469 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fc05b337e1511e0b67f07003e37ed876 in 1643ms, sequenceid=361, compaction requested=true at 1731766645472 (+3 ms)Writing region close event to WAL at 1731766645521 (+49 ms)Running coprocessor post-close hooks at 1731766645524 (+3 ms)Closed at 1731766645524 2024-11-16T14:17:25,525 INFO [RS_CLOSE_REGION-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,525 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=fc05b337e1511e0b67f07003e37ed876, regionState=CLOSED 2024-11-16T14:17:25,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 because future has completed 2024-11-16T14:17:25,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-16T14:17:25,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure fc05b337e1511e0b67f07003e37ed876, server=04e93f2b5fad,41317,1731766448342 in 1.8510 sec 2024-11-16T14:17:25,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-16T14:17:25,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc05b337e1511e0b67f07003e37ed876, UNASSIGN in 1.8540 sec 2024-11-16T14:17:25,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-16T14:17:25,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8580 sec 2024-11-16T14:17:25,531 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731766645531"}]},"ts":"1731766645531"} 2024-11-16T14:17:25,532 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T14:17:25,532 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T14:17:25,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8650 sec 2024-11-16T14:17:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T14:17:25,825 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:17:25,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T14:17:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:17:25,827 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:17:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T14:17:25,828 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:17:25,830 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,832 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C, FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/recovered.edits] 2024-11-16T14:17:25,834 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/2fdc767e9fa0467b8c7d0057672f135f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/2fdc767e9fa0467b8c7d0057672f135f 2024-11-16T14:17:25,835 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3aef8617e6ee4421b2ec998aecbbbbe6 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/3aef8617e6ee4421b2ec998aecbbbbe6 2024-11-16T14:17:25,835 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/9b3ca5a1ab5c4cb9be4a1cd6220db660 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/A/9b3ca5a1ab5c4cb9be4a1cd6220db660 2024-11-16T14:17:25,837 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/16ea84044e1b4bf5a78b88a97d66b168 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/16ea84044e1b4bf5a78b88a97d66b168 2024-11-16T14:17:25,838 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/496a3b83bc544b338b84021586f9a279 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/496a3b83bc544b338b84021586f9a279 2024-11-16T14:17:25,838 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/96830afd3def4010915f7702d5fce1b8 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/B/96830afd3def4010915f7702d5fce1b8 2024-11-16T14:17:25,840 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/0842fc5410654a2696b09923deb8663f to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/0842fc5410654a2696b09923deb8663f 2024-11-16T14:17:25,841 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/67edd4331ebf4b60b5974f52445f644a to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/67edd4331ebf4b60b5974f52445f644a 2024-11-16T14:17:25,841 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/9bbab071b57d4b61ab0023352e07176b to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/C/9bbab071b57d4b61ab0023352e07176b 2024-11-16T14:17:25,843 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/recovered.edits/364.seqid to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876/recovered.edits/364.seqid 2024-11-16T14:17:25,843 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/default/TestAcidGuarantees/fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,843 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T14:17:25,844 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T14:17:25,844 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T14:17:25,846 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111600b9fe7c0fa3477ab2562ad69475c444_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111600b9fe7c0fa3477ab2562ad69475c444_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,847 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164a3177104cdc4358983d0d94abdb9ea9_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164a3177104cdc4358983d0d94abdb9ea9_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,847 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164ee53d4b0ff04d3cb04ae5b4acc68969_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164ee53d4b0ff04d3cb04ae5b4acc68969_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,848 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111659168776a79f4e06905d7b8359b3161f_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111659168776a79f4e06905d7b8359b3161f_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,849 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116594e21dc8b744c2cbf9036ea4ff44794_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116594e21dc8b744c2cbf9036ea4ff44794_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,849 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165cf953ab645247f7b56f1475defb6298_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165cf953ab645247f7b56f1475defb6298_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,850 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111669d34e0c8b3a4d1da72b573dc8b6242f_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111669d34e0c8b3a4d1da72b573dc8b6242f_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,851 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168062aad20b914b44b768f27445a86b97_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168062aad20b914b44b768f27445a86b97_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,851 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116815009f8ebd044498235573e39c9dc31_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116815009f8ebd044498235573e39c9dc31_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,852 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168a441515b4354edc8a542475dc0871f6_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168a441515b4354edc8a542475dc0871f6_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,853 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169b73882563684c1e9ce4d90d78c8b7de_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169b73882563684c1e9ce4d90d78c8b7de_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,854 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a52b42a617a84386ba0e620756cb8a76_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a52b42a617a84386ba0e620756cb8a76_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,854 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a6f6ccc7715a4a209c14a63c1e6ba576_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a6f6ccc7715a4a209c14a63c1e6ba576_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,855 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c2c6177b61a24328aee8fc22430f7e37_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c2c6177b61a24328aee8fc22430f7e37_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,856 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c5538f5763914832b66da310eab687e3_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c5538f5763914832b66da310eab687e3_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,856 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d0bc6a649e2d40a38178705f915d1781_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d0bc6a649e2d40a38178705f915d1781_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,857 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d0fab697d3814ed49e11aa93ccece049_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d0fab697d3814ed49e11aa93ccece049_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,857 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fa93ee75b2d74c10b80777afc4fa72dd_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fa93ee75b2d74c10b80777afc4fa72dd_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,858 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fee22eb748864303961f64a81f237bcd_fc05b337e1511e0b67f07003e37ed876 to hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fee22eb748864303961f64a81f237bcd_fc05b337e1511e0b67f07003e37ed876 2024-11-16T14:17:25,858 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T14:17:25,860 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:17:25,862 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T14:17:25,863 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T14:17:25,864 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:17:25,864 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T14:17:25,864 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731766645864"}]},"ts":"9223372036854775807"} 2024-11-16T14:17:25,865 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T14:17:25,865 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fc05b337e1511e0b67f07003e37ed876, NAME => 'TestAcidGuarantees,,1731766610312.fc05b337e1511e0b67f07003e37ed876.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T14:17:25,865 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T14:17:25,865 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731766645865"}]},"ts":"9223372036854775807"} 2024-11-16T14:17:25,866 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T14:17:25,867 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:17:25,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-16T14:17:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T14:17:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T14:17:25,936 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:17:25,943 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 248), OpenFileDescriptor=445 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=411 (was 381) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4093 (was 4546) 2024-11-16T14:17:25,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T14:17:25,943 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:17:25,943 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:17:25,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:25,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:25,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:17:25,943 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T14:17:25,944 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1336910459, stopped=false 2024-11-16T14:17:25,944 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=04e93f2b5fad,39231,1731766447537 2024-11-16T14:17:25,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T14:17:25,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T14:17:25,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:17:25,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:17:25,996 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:17:25,996 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:17:25,996 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:17:25,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:25,997 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '04e93f2b5fad,41317,1731766448342' ***** 2024-11-16T14:17:25,997 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T14:17:25,997 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:17:26,000 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:17:26,000 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(878): Closing user regions 2024-11-16T14:17:26,001 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T14:17:26,002 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T14:17:26,002 INFO [RS:0;04e93f2b5fad:41317 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T14:17:26,002 INFO [RS:0;04e93f2b5fad:41317 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T14:17:26,002 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(959): stopping server 04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:26,002 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T14:17:26,002 INFO [RS:0;04e93f2b5fad:41317 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;04e93f2b5fad:41317. 2024-11-16T14:17:26,002 DEBUG [RS:0;04e93f2b5fad:41317 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:17:26,002 DEBUG [RS:0;04e93f2b5fad:41317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:26,002 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T14:17:26,002 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T14:17:26,002 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T14:17:26,002 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T14:17:26,003 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T14:17:26,003 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T14:17:26,003 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:17:26,003 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T14:17:26,003 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T14:17:26,003 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T14:17:26,003 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T14:17:26,003 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T14:17:26,003 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-16T14:17:26,011 INFO [regionserver/04e93f2b5fad:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T14:17:26,023 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/.tmp/info/a40b8206724b4dfdb4b6d4ba4cd0e44b is 95, key is TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7./info:/1731766487829/DeleteFamily/seqid=0 2024-11-16T14:17:26,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742476_1652 (size=6514) 2024-11-16T14:17:26,203 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:17:26,403 INFO [regionserver/04e93f2b5fad:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T14:17:26,403 INFO [regionserver/04e93f2b5fad:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T14:17:26,403 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:17:26,427 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/.tmp/info/a40b8206724b4dfdb4b6d4ba4cd0e44b 2024-11-16T14:17:26,443 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/.tmp/ns/1115508841204acb91f0c88b5d2b0ffb is 93, key is TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7./ns:/1731766487783/DeleteFamily/seqid=0 2024-11-16T14:17:26,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742477_1653 (size=5906) 2024-11-16T14:17:26,449 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/.tmp/ns/1115508841204acb91f0c88b5d2b0ffb 2024-11-16T14:17:26,464 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/.tmp/rep_barrier/50e7c16ec11441778c2187176a37826b is 102, key is TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7./rep_barrier:/1731766487783/DeleteFamily/seqid=0 2024-11-16T14:17:26,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742478_1654 (size=6025) 2024-11-16T14:17:26,604 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:17:26,804 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:17:26,868 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/.tmp/rep_barrier/50e7c16ec11441778c2187176a37826b 2024-11-16T14:17:26,888 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/.tmp/table/2167f7d5ba4a4034803b1e92c3c1c939 is 96, key is TestAcidGuarantees,,1731766453934.fd241767f9d916c6c1f6b781eb2b8ff7./table:/1731766487783/DeleteFamily/seqid=0 2024-11-16T14:17:26,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742479_1655 (size=5936) 2024-11-16T14:17:27,004 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:17:27,204 DEBUG [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:17:27,291 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/.tmp/table/2167f7d5ba4a4034803b1e92c3c1c939 2024-11-16T14:17:27,299 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/.tmp/info/a40b8206724b4dfdb4b6d4ba4cd0e44b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/info/a40b8206724b4dfdb4b6d4ba4cd0e44b 2024-11-16T14:17:27,301 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/info/a40b8206724b4dfdb4b6d4ba4cd0e44b, entries=12, sequenceid=90, filesize=6.4 K 2024-11-16T14:17:27,302 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/.tmp/ns/1115508841204acb91f0c88b5d2b0ffb as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/ns/1115508841204acb91f0c88b5d2b0ffb 2024-11-16T14:17:27,304 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/ns/1115508841204acb91f0c88b5d2b0ffb, entries=8, sequenceid=90, filesize=5.8 K 2024-11-16T14:17:27,304 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/.tmp/rep_barrier/50e7c16ec11441778c2187176a37826b as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/rep_barrier/50e7c16ec11441778c2187176a37826b 2024-11-16T14:17:27,307 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/rep_barrier/50e7c16ec11441778c2187176a37826b, entries=6, sequenceid=90, filesize=5.9 K 2024-11-16T14:17:27,307 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/.tmp/table/2167f7d5ba4a4034803b1e92c3c1c939 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/table/2167f7d5ba4a4034803b1e92c3c1c939 2024-11-16T14:17:27,309 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/table/2167f7d5ba4a4034803b1e92c3c1c939, entries=7, sequenceid=90, filesize=5.8 K 2024-11-16T14:17:27,310 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1307ms, sequenceid=90, compaction requested=false 2024-11-16T14:17:27,313 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-16T14:17:27,313 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T14:17:27,313 INFO [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T14:17:27,313 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731766646003Running coprocessor pre-close hooks at 1731766646003Disabling compacts and flushes for region at 1731766646003Disabling writes for close at 1731766646003Obtaining lock to block concurrent updates at 1731766646003Preparing flush snapshotting stores in 1588230740 at 1731766646003Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731766646004 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731766646004Flushing 1588230740/info: creating writer at 1731766646004Flushing 1588230740/info: appending metadata at 1731766646023 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731766646023Flushing 1588230740/ns: creating writer at 1731766646430 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731766646442 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1731766646442Flushing 1588230740/rep_barrier: creating writer at 1731766646452 (+10 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731766646464 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731766646464Flushing 1588230740/table: creating writer at 1731766646871 (+407 ms)Flushing 1588230740/table: appending metadata at 1731766646888 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731766646888Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@208f3be4: reopening flushed file at 1731766647298 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74ba25cc: reopening flushed file at 1731766647301 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cbdacaf: reopening flushed file at 1731766647304 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2426ae5e: reopening flushed file at 1731766647307 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1307ms, sequenceid=90, compaction requested=false at 1731766647310 (+3 ms)Writing region close event to WAL at 1731766647311 (+1 ms)Running coprocessor post-close hooks at 1731766647313 (+2 ms)Closed at 1731766647313 2024-11-16T14:17:27,314 DEBUG [RS_CLOSE_META-regionserver/04e93f2b5fad:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T14:17:27,404 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(976): stopping server 04e93f2b5fad,41317,1731766448342; all regions closed. 2024-11-16T14:17:27,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741834_1010 (size=25344) 2024-11-16T14:17:27,411 DEBUG [RS:0;04e93f2b5fad:41317 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/oldWALs 2024-11-16T14:17:27,411 INFO [RS:0;04e93f2b5fad:41317 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 04e93f2b5fad%2C41317%2C1731766448342.meta:.meta(num 1731766453474) 2024-11-16T14:17:27,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741833_1009 (size=17456793) 2024-11-16T14:17:27,415 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/WALs/04e93f2b5fad,41317,1731766448342/04e93f2b5fad%2C41317%2C1731766448342.1731766452578 not finished, retry = 0 2024-11-16T14:17:27,517 DEBUG [RS:0;04e93f2b5fad:41317 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/oldWALs 2024-11-16T14:17:27,517 INFO [RS:0;04e93f2b5fad:41317 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 04e93f2b5fad%2C41317%2C1731766448342:(num 1731766452578) 2024-11-16T14:17:27,517 DEBUG [RS:0;04e93f2b5fad:41317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:17:27,517 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T14:17:27,517 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T14:17:27,517 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.ChoreService(370): Chore service for: regionserver/04e93f2b5fad:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T14:17:27,517 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T14:17:27,517 INFO [regionserver/04e93f2b5fad:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T14:17:27,518 INFO [RS:0;04e93f2b5fad:41317 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41317 2024-11-16T14:17:27,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/04e93f2b5fad,41317,1731766448342 2024-11-16T14:17:27,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T14:17:27,543 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T14:17:27,544 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [04e93f2b5fad,41317,1731766448342] 2024-11-16T14:17:27,564 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/04e93f2b5fad,41317,1731766448342 already deleted, retry=false 2024-11-16T14:17:27,564 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 04e93f2b5fad,41317,1731766448342 expired; onlineServers=0 2024-11-16T14:17:27,564 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '04e93f2b5fad,39231,1731766447537' ***** 2024-11-16T14:17:27,564 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T14:17:27,564 INFO [M:0;04e93f2b5fad:39231 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T14:17:27,564 INFO [M:0;04e93f2b5fad:39231 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T14:17:27,565 DEBUG [M:0;04e93f2b5fad:39231 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T14:17:27,565 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T14:17:27,565 DEBUG [M:0;04e93f2b5fad:39231 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T14:17:27,565 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster-HFileCleaner.small.0-1731766452175 {}] cleaner.HFileCleaner(306): Exit Thread[master/04e93f2b5fad:0:becomeActiveMaster-HFileCleaner.small.0-1731766452175,5,FailOnTimeoutGroup] 2024-11-16T14:17:27,565 DEBUG [master/04e93f2b5fad:0:becomeActiveMaster-HFileCleaner.large.0-1731766452171 {}] cleaner.HFileCleaner(306): Exit Thread[master/04e93f2b5fad:0:becomeActiveMaster-HFileCleaner.large.0-1731766452171,5,FailOnTimeoutGroup] 2024-11-16T14:17:27,565 INFO [M:0;04e93f2b5fad:39231 {}] hbase.ChoreService(370): Chore service for: master/04e93f2b5fad:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T14:17:27,565 INFO [M:0;04e93f2b5fad:39231 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T14:17:27,565 DEBUG [M:0;04e93f2b5fad:39231 {}] master.HMaster(1795): Stopping service threads 2024-11-16T14:17:27,565 INFO [M:0;04e93f2b5fad:39231 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T14:17:27,565 INFO [M:0;04e93f2b5fad:39231 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:17:27,565 ERROR [M:0;04e93f2b5fad:39231 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:43297 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:43297,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-16T14:17:27,566 INFO [M:0;04e93f2b5fad:39231 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T14:17:27,566 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T14:17:27,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T14:17:27,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:17:27,575 DEBUG [M:0;04e93f2b5fad:39231 {}] zookeeper.ZKUtil(347): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T14:17:27,575 WARN [M:0;04e93f2b5fad:39231 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T14:17:27,576 INFO [M:0;04e93f2b5fad:39231 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/.lastflushedseqids 2024-11-16T14:17:27,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742480_1656 (size=72) 2024-11-16T14:17:27,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:17:27,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41317-0x10143fee65f0001, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:17:27,665 INFO [RS:0;04e93f2b5fad:41317 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T14:17:27,665 INFO [RS:0;04e93f2b5fad:41317 {}] regionserver.HRegionServer(1031): Exiting; stopping=04e93f2b5fad,41317,1731766448342; zookeeper connection closed. 2024-11-16T14:17:27,665 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1968f5c2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1968f5c2 2024-11-16T14:17:27,666 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-16T14:17:27,873 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T14:17:27,873 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T14:17:27,874 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T14:17:27,987 INFO [M:0;04e93f2b5fad:39231 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T14:17:27,987 INFO [M:0;04e93f2b5fad:39231 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T14:17:27,987 DEBUG [M:0;04e93f2b5fad:39231 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T14:17:27,987 INFO [M:0;04e93f2b5fad:39231 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:17:27,987 DEBUG [M:0;04e93f2b5fad:39231 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:17:27,987 DEBUG [M:0;04e93f2b5fad:39231 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T14:17:27,987 DEBUG [M:0;04e93f2b5fad:39231 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:17:27,987 INFO [M:0;04e93f2b5fad:39231 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=746.61 KB heapSize=912.53 KB 2024-11-16T14:17:28,002 DEBUG [M:0;04e93f2b5fad:39231 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1882a7d74bd14b629e4809b3ab6d107f is 82, key is hbase:meta,,1/info:regioninfo/1731766453557/Put/seqid=0 2024-11-16T14:17:28,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742481_1657 (size=5672) 2024-11-16T14:17:28,405 INFO [M:0;04e93f2b5fad:39231 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2037 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1882a7d74bd14b629e4809b3ab6d107f 2024-11-16T14:17:28,424 DEBUG [M:0;04e93f2b5fad:39231 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/73a91b29a0f84596a9ac5d4912223d06 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1731766613396/Put/seqid=0 2024-11-16T14:17:28,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742482_1658 (size=45355) 2024-11-16T14:17:28,828 INFO [M:0;04e93f2b5fad:39231 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746.06 KB at sequenceid=2037 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/73a91b29a0f84596a9ac5d4912223d06 2024-11-16T14:17:28,831 INFO [M:0;04e93f2b5fad:39231 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 73a91b29a0f84596a9ac5d4912223d06 2024-11-16T14:17:28,847 DEBUG [M:0;04e93f2b5fad:39231 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/916375a9072b449f9e9ccbcac05601ad is 69, key is 04e93f2b5fad,41317,1731766448342/rs:state/1731766452238/Put/seqid=0 2024-11-16T14:17:28,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742483_1659 (size=5156) 2024-11-16T14:17:29,250 INFO [M:0;04e93f2b5fad:39231 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2037 (bloomFilter=true), to=hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/916375a9072b449f9e9ccbcac05601ad 2024-11-16T14:17:29,253 DEBUG [M:0;04e93f2b5fad:39231 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1882a7d74bd14b629e4809b3ab6d107f as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1882a7d74bd14b629e4809b3ab6d107f 2024-11-16T14:17:29,256 INFO [M:0;04e93f2b5fad:39231 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1882a7d74bd14b629e4809b3ab6d107f, entries=8, sequenceid=2037, filesize=5.5 K 2024-11-16T14:17:29,257 DEBUG [M:0;04e93f2b5fad:39231 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/73a91b29a0f84596a9ac5d4912223d06 as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/73a91b29a0f84596a9ac5d4912223d06 2024-11-16T14:17:29,259 INFO [M:0;04e93f2b5fad:39231 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 73a91b29a0f84596a9ac5d4912223d06 2024-11-16T14:17:29,259 INFO [M:0;04e93f2b5fad:39231 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/73a91b29a0f84596a9ac5d4912223d06, entries=170, sequenceid=2037, filesize=44.3 K 2024-11-16T14:17:29,259 DEBUG [M:0;04e93f2b5fad:39231 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/916375a9072b449f9e9ccbcac05601ad as hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/916375a9072b449f9e9ccbcac05601ad 2024-11-16T14:17:29,261 INFO [M:0;04e93f2b5fad:39231 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43297/user/jenkins/test-data/4d0b1c90-2483-4694-f2a1-90768d370adc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/916375a9072b449f9e9ccbcac05601ad, entries=1, sequenceid=2037, filesize=5.0 K 2024-11-16T14:17:29,262 INFO [M:0;04e93f2b5fad:39231 {}] regionserver.HRegion(3140): Finished flush of dataSize ~746.61 KB/764532, heapSize ~912.23 KB/934128, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=2037, compaction requested=false 2024-11-16T14:17:29,263 INFO [M:0;04e93f2b5fad:39231 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:17:29,263 DEBUG [M:0;04e93f2b5fad:39231 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731766647987Disabling compacts and flushes for region at 1731766647987Disabling writes for close at 1731766647987Obtaining lock to block concurrent updates at 1731766647987Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731766647987Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=764532, getHeapSize=934368, getOffHeapSize=0, getCellsCount=2232 at 1731766647987Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731766647988 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731766647988Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731766648001 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731766648001Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731766648408 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731766648423 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731766648424 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731766648831 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731766648846 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731766648846Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@195fe1ec: reopening flushed file at 1731766649253 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@508eb45c: reopening flushed file at 1731766649256 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6de4df07: reopening flushed file at 1731766649259 (+3 ms)Finished flush of dataSize ~746.61 KB/764532, heapSize ~912.23 KB/934128, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=2037, compaction requested=false at 1731766649262 (+3 ms)Writing region close event to WAL at 1731766649263 (+1 ms)Closed at 1731766649263 2024-11-16T14:17:29,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741830_1006 (size=898136) 2024-11-16T14:17:29,265 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T14:17:29,265 INFO [M:0;04e93f2b5fad:39231 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T14:17:29,265 INFO [M:0;04e93f2b5fad:39231 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39231 2024-11-16T14:17:29,266 INFO [M:0;04e93f2b5fad:39231 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T14:17:29,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:17:29,383 INFO [M:0;04e93f2b5fad:39231 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T14:17:29,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x10143fee65f0000, quorum=127.0.0.1:50174, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:17:29,388 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:17:29,390 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T14:17:29,390 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T14:17:29,390 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T14:17:29,390 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/hadoop.log.dir/,STOPPED} 2024-11-16T14:17:29,392 WARN [BP-517253520-172.17.0.2-1731766443902 heartbeating to localhost/127.0.0.1:43297 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T14:17:29,392 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T14:17:29,392 WARN [BP-517253520-172.17.0.2-1731766443902 heartbeating to localhost/127.0.0.1:43297 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-517253520-172.17.0.2-1731766443902 (Datanode Uuid d94a9d10-33bc-48ed-8e36-884d4c3230f4) service to localhost/127.0.0.1:43297 2024-11-16T14:17:29,392 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T14:17:29,395 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/cluster_786ab5e4-0d7c-0505-b9b1-bd85905ca37b/data/data1/current/BP-517253520-172.17.0.2-1731766443902 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T14:17:29,395 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/cluster_786ab5e4-0d7c-0505-b9b1-bd85905ca37b/data/data2/current/BP-517253520-172.17.0.2-1731766443902 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T14:17:29,396 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T14:17:29,402 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T14:17:29,402 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T14:17:29,402 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T14:17:29,403 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T14:17:29,403 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4bd7035-8f47-da56-383c-0e1e3ee28164/hadoop.log.dir/,STOPPED} 2024-11-16T14:17:29,416 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T14:17:29,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down